Add SubColumn SQL support for integer and float types (#18077)
This syncs the SubColumn implementation used by the paper artifact, including FLOAT scaling support and SQL/schema tests.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index c8299c5..8b5da9a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -362,7 +362,7 @@
String datatypeString =
props.get(IoTDBConstant.COLUMN_TIMESERIES_DATATYPE.toLowerCase()).toUpperCase();
try {
- createTimeSeriesStatement.setDataType(TSDataType.valueOf(datatypeString));
+ createTimeSeriesStatement.setDataType(parseTimeSeriesDataType(datatypeString));
props.remove(IoTDBConstant.COLUMN_TIMESERIES_DATATYPE.toLowerCase());
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported datatype: %s", datatypeString));
@@ -3428,7 +3428,7 @@
}
String dataTypeString = ctx.dataType.getText().toUpperCase();
try {
- dataType = TSDataType.valueOf(dataTypeString);
+ dataType = parseTimeSeriesDataType(dataTypeString);
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported datatype: %s", dataTypeString));
}
@@ -3436,6 +3436,13 @@
return dataType;
}
+ private TSDataType parseTimeSeriesDataType(String dataTypeString) {
+ if ("INTEGER".equals(dataTypeString)) {
+ return TSDataType.INT32;
+ }
+ return TSDataType.valueOf(dataTypeString);
+ }
+
@Override
public Statement visitShowSchemaTemplates(IoTDBSqlParser.ShowSchemaTemplatesContext ctx) {
return new ShowSchemaTemplateStatement();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
index efae314..e489fa2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -56,6 +56,7 @@
intSet.add(TSEncoding.CHIMP);
intSet.add(TSEncoding.SPRINTZ);
intSet.add(TSEncoding.RLBE);
+ intSet.add(TSEncoding.SUBCOLUMN);
schemaChecker.put(TSDataType.INT32, intSet);
schemaChecker.put(TSDataType.INT64, intSet);
@@ -69,9 +70,12 @@
floatSet.add(TSEncoding.CHIMP);
floatSet.add(TSEncoding.SPRINTZ);
floatSet.add(TSEncoding.RLBE);
+ floatSet.add(TSEncoding.SUBCOLUMN);
schemaChecker.put(TSDataType.FLOAT, floatSet);
- schemaChecker.put(TSDataType.DOUBLE, floatSet);
+ Set<TSEncoding> doubleSet = new HashSet<>(floatSet);
+ doubleSet.remove(TSEncoding.SUBCOLUMN);
+ schemaChecker.put(TSDataType.DOUBLE, doubleSet);
Set<TSEncoding> textSet = new HashSet<>();
textSet.add(TSEncoding.PLAIN);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
index ff0ec60..23bbc39 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
@@ -281,6 +281,32 @@
}
@Test
+ public void testCreateTimeSeriesWithSubColumnEncodingSql() throws IllegalPathException {
+ Statement parsed =
+ StatementGenerator.createStatement(
+ "create timeseries root.temperature INTEGER encoding=SubColumn",
+ ZonedDateTime.now().getOffset());
+
+ CreateTimeSeriesStatement statement = (CreateTimeSeriesStatement) parsed;
+ assertEquals(new PartialPath("root.temperature"), statement.getPath());
+ assertEquals(TSDataType.INT32, statement.getDataType());
+ assertEquals(TSEncoding.SUBCOLUMN, statement.getEncoding());
+ }
+
+ @Test
+ public void testCreateFloatTimeSeriesWithSubColumnEncodingSql() throws IllegalPathException {
+ Statement parsed =
+ StatementGenerator.createStatement(
+ "create timeseries root.temperature FLOAT encoding=SubColumn",
+ ZonedDateTime.now().getOffset());
+
+ CreateTimeSeriesStatement statement = (CreateTimeSeriesStatement) parsed;
+ assertEquals(new PartialPath("root.temperature"), statement.getPath());
+ assertEquals(TSDataType.FLOAT, statement.getDataType());
+ assertEquals(TSEncoding.SUBCOLUMN, statement.getEncoding());
+ }
+
+ @Test
public void testCreateAlignedTimeSeries() throws IllegalPathException {
TSCreateAlignedTimeseriesReq req =
new TSCreateAlignedTimeseriesReq(
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
index 7c76d5b..b21d3a6 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java
@@ -75,4 +75,16 @@
// do nothing
}
}
+
+ @Test
+ public void checkSubColumnEncodingForInteger() throws MetadataException {
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.INT32, TSEncoding.SUBCOLUMN);
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.FLOAT, TSEncoding.SUBCOLUMN);
+ try {
+ SchemaUtils.checkDataTypeWithEncoding(TSDataType.TEXT, TSEncoding.SUBCOLUMN);
+ Assert.fail("expect exception");
+ } catch (MetadataException e) {
+ // do nothing
+ }
+ }
}
diff --git a/iotdb-core/tsfile/pom.xml b/iotdb-core/tsfile/pom.xml
index 39f45ce..21546c9 100644
--- a/iotdb-core/tsfile/pom.xml
+++ b/iotdb-core/tsfile/pom.xml
@@ -191,7 +191,8 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
- <version>1.21</version> <!-- 请检查是否有更新的版本 -->
+ <version>1.21</version>
+ <!-- 请检查是否有更新的版本 -->
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -199,20 +200,28 @@
<version>3.6.1</version>
</dependency>
<dependency>
+ <groupId>net.java.dev.jna</groupId>
+ <artifactId>jna-platform</artifactId>
+ <version>${jna.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- ELF bench: bit streams from elf@elf (dsiutils fastutil) -->
+ <dependency>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>dsiutils</artifactId>
+ <version>2.7.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>24.0.1</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-antlr</artifactId>
- <version>0.14.0-SNAPSHOT</version>
- </dependency>
- <dependency>
<groupId>me.lemire.integercompression</groupId>
<artifactId>JavaFastPFOR</artifactId>
- <version>[0.1,)</version>
+ <version>0.3.13</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
index 7a01964..1333bcb 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -161,6 +161,18 @@
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
+ case SUBCOLUMN:
+ switch (dataType) {
+ case INT32:
+ return new IntSubColumnDecoder();
+ case INT64:
+ case VECTOR:
+ return new LongSubColumnDecoder();
+ case FLOAT:
+ return new FloatSubColumnDecoder();
+ default:
+ throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
+ }
default:
throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType));
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java
new file mode 100644
index 0000000..48a3718
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FloatSubColumnDecoder.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for FLOAT values encoded by FloatSubColumnEncoder. */
+public class FloatSubColumnDecoder extends Decoder {
+
+ private float[] values;
+ private int index;
+
+ public FloatSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public float readFloat(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int precision = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int blockSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ double scale = Math.pow(10, precision);
+ int[] scaledValues = new int[count];
+ int offset = 0;
+ while (offset < count) {
+ int length = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int min = buffer.getInt();
+ int bitWidth = buffer.get() & 0xFF;
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int byteIndex = 0; byteIndex < bytesPerPlane; byteIndex++) {
+ int mask = buffer.get() & 0xFF;
+ for (int j = 0; j < Byte.SIZE; j++) {
+ int localIndex = (byteIndex << 3) + j;
+ if (localIndex < length && ((mask >>> j) & 1) != 0) {
+ scaledValues[offset + localIndex] |= 1 << bit;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ scaledValues[offset + i] += min;
+ }
+ offset += Math.min(blockSize, length);
+ }
+ values = new float[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = (float) (scaledValues[i] / scale);
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java
new file mode 100644
index 0000000..6178f5b
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/IntSubColumnDecoder.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for INT32 values encoded by {@link org.apache.iotdb.tsfile.encoding.encoder.IntSubColumnEncoder}. */
+public class IntSubColumnDecoder extends Decoder {
+
+ private int[] values;
+ private int index;
+
+ public IntSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public int readInt(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int blockSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ values = new int[count];
+ int offset = 0;
+ while (offset < count) {
+ int length = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ int min = buffer.getInt();
+ int bitWidth = buffer.get() & 0xFF;
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int byteIndex = 0; byteIndex < bytesPerPlane; byteIndex++) {
+ int mask = buffer.get() & 0xFF;
+ for (int j = 0; j < Byte.SIZE; j++) {
+ int localIndex = (byteIndex << 3) + j;
+ if (localIndex < length && ((mask >>> j) & 1) != 0) {
+ values[offset + localIndex] |= 1 << bit;
+ }
+ }
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ values[offset + i] += min;
+ }
+ offset += Math.min(blockSize, length);
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java
new file mode 100644
index 0000000..1d553e1
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/LongSubColumnDecoder.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.nio.ByteBuffer;
+
+/** Decoder for INT64 values encoded by {@link org.apache.iotdb.tsfile.encoding.encoder.LongSubColumnEncoder}. */
+public class LongSubColumnDecoder extends Decoder {
+
+ private long[] values;
+ private int index;
+
+ public LongSubColumnDecoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public long readLong(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values[index++];
+ }
+
+ @Override
+ public boolean hasNext(ByteBuffer buffer) {
+ ensureLoaded(buffer);
+ return values != null && index < values.length;
+ }
+
+ @Override
+ public void reset() {
+ values = null;
+ index = 0;
+ }
+
+ private void ensureLoaded(ByteBuffer buffer) {
+ if (values != null || !buffer.hasRemaining()) {
+ return;
+ }
+ int count = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
+ values = new long[count];
+ for (int i = 0; i < count; i++) {
+ values[i] = buffer.getLong();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java
new file mode 100644
index 0000000..f989b53
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FloatSubColumnEncoder.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for FLOAT values using decimal scaling and sub-column bit-plane packing. */
+public class FloatSubColumnEncoder extends Encoder {
+
+ private static final int BLOCK_SIZE = 128;
+ private static final int MAX_FLOAT_DECIMAL_PRECISION = 6;
+
+ private final List<Float> values = new ArrayList<>();
+
+ public FloatSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ int precision = choosePrecision();
+ double scale = Math.pow(10, precision);
+ int[] scaledValues = new int[values.size()];
+ for (int i = 0; i < values.size(); i++) {
+ scaledValues[i] = (int) Math.round(values.get(i) * scale);
+ }
+
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(scaledValues.length, out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(precision, out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(BLOCK_SIZE, out);
+ for (int offset = 0; offset < scaledValues.length; offset += BLOCK_SIZE) {
+ int length = Math.min(BLOCK_SIZE, scaledValues.length - offset);
+ writeBlock(out, scaledValues, offset, length);
+ }
+ values.clear();
+ }
+
+ private int choosePrecision() {
+ int precision = 0;
+ for (float value : values) {
+ precision = Math.max(precision, getDecimalPrecision(value));
+ }
+ precision = Math.min(precision, MAX_FLOAT_DECIMAL_PRECISION);
+ while (precision > 0 && !fitsInInt(precision)) {
+ precision--;
+ }
+ return precision;
+ }
+
+ private boolean fitsInInt(int precision) {
+ double scale = Math.pow(10, precision);
+ for (float value : values) {
+ double scaled = Math.rint(value * scale);
+ if (scaled < Integer.MIN_VALUE || scaled > Integer.MAX_VALUE) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static int getDecimalPrecision(float value) {
+ BigDecimal decimal = new BigDecimal(Float.toString(value)).stripTrailingZeros();
+ return Math.max(0, decimal.scale());
+ }
+
+ private static void writeBlock(
+ ByteArrayOutputStream out, int[] scaledValues, int offset, int length) throws IOException {
+ int min = scaledValues[offset];
+ long maxDelta = 0;
+ for (int i = 0; i < length; i++) {
+ int value = scaledValues[offset + i];
+ if (value < min) {
+ min = value;
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) scaledValues[offset + i] - min;
+ if (delta > maxDelta) {
+ maxDelta = delta;
+ }
+ }
+
+ int bitWidth = maxDelta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(maxDelta);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(length, out);
+ writeInt(out, min);
+ out.write(bitWidth);
+ if (bitWidth == 0) {
+ return;
+ }
+
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ byte[] plane = new byte[bytesPerPlane];
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int i = 0; i < bytesPerPlane; i++) {
+ plane[i] = 0;
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) scaledValues[offset + i] - min;
+ if (((delta >>> bit) & 1L) != 0) {
+ plane[i >>> 3] |= (byte) (1 << (i & 7));
+ }
+ }
+ out.write(plane);
+ }
+ }
+
+ private static void writeInt(ByteArrayOutputStream out, int value) {
+ out.write((value >>> 24) & 0xFF);
+ out.write((value >>> 16) & 0xFF);
+ out.write((value >>> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Integer.BYTES + 1;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * (Integer.BYTES + 1) + 16;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java
new file mode 100644
index 0000000..c4c6c8a
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/IntSubColumnEncoder.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for INT32 values using block-level sub-column bit-plane packing. */
+public class IntSubColumnEncoder extends Encoder {
+
+ private static final int BLOCK_SIZE = 128;
+
+ private final List<Integer> values = new ArrayList<>();
+
+ public IntSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(values.size(), out);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(BLOCK_SIZE, out);
+ for (int offset = 0; offset < values.size(); offset += BLOCK_SIZE) {
+ int length = Math.min(BLOCK_SIZE, values.size() - offset);
+ writeBlock(out, offset, length);
+ }
+ values.clear();
+ }
+
+ private void writeBlock(ByteArrayOutputStream out, int offset, int length) throws IOException {
+ int min = values.get(offset);
+ long maxDelta = 0;
+ for (int i = 0; i < length; i++) {
+ int value = values.get(offset + i);
+ if (value < min) {
+ min = value;
+ }
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) values.get(offset + i) - min;
+ if (delta > maxDelta) {
+ maxDelta = delta;
+ }
+ }
+
+ int bitWidth = maxDelta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(maxDelta);
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(length, out);
+ writeInt(out, min);
+ out.write(bitWidth);
+ if (bitWidth == 0) {
+ return;
+ }
+
+ int bytesPerPlane = (length + Byte.SIZE - 1) / Byte.SIZE;
+ byte[] plane = new byte[bytesPerPlane];
+ for (int bit = 0; bit < bitWidth; bit++) {
+ for (int i = 0; i < bytesPerPlane; i++) {
+ plane[i] = 0;
+ }
+ for (int i = 0; i < length; i++) {
+ long delta = (long) values.get(offset + i) - min;
+ if (((delta >>> bit) & 1L) != 0) {
+ plane[i >>> 3] |= (byte) (1 << (i & 7));
+ }
+ }
+ out.write(plane);
+ }
+ }
+
+ private static void writeInt(ByteArrayOutputStream out, int value) {
+ out.write((value >>> 24) & 0xFF);
+ out.write((value >>> 16) & 0xFF);
+ out.write((value >>> 8) & 0xFF);
+ out.write(value & 0xFF);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Integer.BYTES + 1;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * (Integer.BYTES + 1) + 16;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java
new file mode 100644
index 0000000..f502947
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/LongSubColumnEncoder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.utils.ReadWriteForEncodingUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Encoder for INT64 values. This keeps SUBCOLUMN schema support lossless for long series. */
+public class LongSubColumnEncoder extends Encoder {
+
+ private final List<Long> values = new ArrayList<>();
+
+ public LongSubColumnEncoder() {
+ super(TSEncoding.SUBCOLUMN);
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ ReadWriteForEncodingUtils.writeUnsignedVarInt(values.size(), out);
+ for (long value : values) {
+ writeLong(out, value);
+ }
+ values.clear();
+ }
+
+ private static void writeLong(ByteArrayOutputStream out, long value) {
+ for (int shift = 56; shift >= 0; shift -= 8) {
+ out.write((int) ((value >>> shift) & 0xFF));
+ }
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return Long.BYTES;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return (long) values.size() * Long.BYTES + 8;
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
index 453aadf..368bab1 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -78,6 +78,8 @@
return new Sprintz();
case RLBE:
return new RLBE();
+ case SUBCOLUMN:
+ return new SubColumn();
default:
throw new UnsupportedOperationException(type.toString());
}
@@ -353,6 +355,27 @@
}
}
+ public static class SubColumn extends TSEncodingBuilder {
+ @Override
+ public Encoder getEncoder(TSDataType type) {
+ switch (type) {
+ case INT32:
+ return new IntSubColumnEncoder();
+ case INT64:
+ return new LongSubColumnEncoder();
+ case FLOAT:
+ return new FloatSubColumnEncoder();
+ default:
+ throw new UnSupportedDataTypeException("SUBCOLUMN doesn't support data type: " + type);
+ }
+ }
+
+ @Override
+ public void initFromProps(Map<String, String> props) {
+ // do nothing
+ }
+ }
+
public static class Dictionary extends TSEncodingBuilder {
@Override
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
index 56cecda..7c2e547 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -34,7 +34,8 @@
FREQ((byte) 10),
CHIMP((byte) 11),
SPRINTZ((byte) 12),
- RLBE((byte) 13);
+ RLBE((byte) 13),
+ SUBCOLUMN((byte) 14);
private final byte type;
TSEncoding(byte type) {
@@ -79,6 +80,8 @@
return TSEncoding.SPRINTZ;
case 13:
return TSEncoding.RLBE;
+ case 14:
+ return TSEncoding.SUBCOLUMN;
default:
throw new IllegalArgumentException("Invalid input: " + encoding);
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java
new file mode 100644
index 0000000..c66b177
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/compress/CSVCompressTest.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile.compress;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class CSVCompressTest {
+
+ private static final String PARENT_DIR = "D://github/xjz17/subcolumn/";
+ private static final String INPUT_PARENT_DIR = PARENT_DIR + "dataset/";
+ private static final String OUTPUT_PARENT_DIR = PARENT_DIR + "result/";
+ private static final String[] HEADERS = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Original Size",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+
+ private static class CompressionBenchmark {
+ private final String algorithm;
+ private final String outputFileName;
+ private final Supplier<ICompressor> compressorSupplier;
+ private final Supplier<IUnCompressor> unCompressorSupplier;
+ private CsvWriter writer;
+
+ private CompressionBenchmark(
+ String algorithm,
+ String outputFileName,
+ Supplier<ICompressor> compressorSupplier,
+ Supplier<IUnCompressor> unCompressorSupplier) {
+ this.algorithm = algorithm;
+ this.outputFileName = outputFileName;
+ this.compressorSupplier = compressorSupplier;
+ this.unCompressorSupplier = unCompressorSupplier;
+ }
+ }
+
+ private static class BenchmarkResult {
+ private final long encodeTime;
+ private final long decodeTime;
+ private final int compressedSize;
+ private final double compressionRatio;
+
+ private BenchmarkResult(
+ long encodeTime, long decodeTime, int compressedSize, double compressionRatio) {
+ this.encodeTime = encodeTime;
+ this.decodeTime = decodeTime;
+ this.compressedSize = compressedSize;
+ this.compressionRatio = compressionRatio;
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+
+ int REPEAT_TIME = 100;
+ // REPEAT_TIME = 500;
+ // REPEAT_TIME = 10;
+
+ Files.createDirectories(Paths.get(OUTPUT_PARENT_DIR));
+
+ File directory = new File(INPUT_PARENT_DIR);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null || csvFiles.length == 0) {
+ return;
+ }
+ Arrays.sort(csvFiles, Comparator.comparing(File::getName));
+
+ List<CompressionBenchmark> benchmarks = new ArrayList<>();
+ benchmarks.add(
+ new CompressionBenchmark(
+ "GZIP",
+ "compress_gzip.csv",
+ ICompressor.GZIPCompressor::new,
+ IUnCompressor.GZIPUnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "LZ4",
+ "compress_lz4.csv",
+ ICompressor.IOTDBLZ4Compressor::new,
+ IUnCompressor.LZ4UnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "SNAPPY",
+ "compress_snappy.csv",
+ ICompressor.SnappyCompressor::new,
+ IUnCompressor.SnappyUnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "LZMA2",
+ "compress_lzma2.csv",
+ ICompressor.LZMA2Compressor::new,
+ IUnCompressor.LZMA2UnCompressor::new));
+ benchmarks.add(
+ new CompressionBenchmark(
+ "ZSTD",
+ "compress_zstd.csv",
+ ICompressor.ZstdCompressor::new,
+ IUnCompressor.ZstdUnCompressor::new));
+
+ try {
+ for (CompressionBenchmark benchmark : benchmarks) {
+ benchmark.writer =
+ new CsvWriter(
+ OUTPUT_PARENT_DIR + benchmark.outputFileName, ',', StandardCharsets.UTF_8);
+ benchmark.writer.setRecordDelimiter('\n');
+ benchmark.writer.writeRecord(HEADERS);
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ int[] data;
+ try (InputStream inputStream = Files.newInputStream(file.toPath())) {
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> floatData = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String value = loader.getValues()[0];
+ if (value.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(value);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ floatData.add(Float.valueOf(value));
+ }
+ loader.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ data = new int[floatData.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < floatData.size(); i++) {
+ data[i] = (int) (floatData.get(i) * maxMul);
+ }
+ }
+
+ if (data.length == 0) {
+ continue;
+ }
+
+ ByteBuffer inputBuffer = ByteBuffer.allocate(data.length * Integer.BYTES);
+ for (int value : data) {
+ inputBuffer.putInt(value);
+ }
+ byte[] inputBytes = inputBuffer.array();
+
+ for (CompressionBenchmark benchmark : benchmarks) {
+ ICompressor compressor = benchmark.compressorSupplier.get();
+ IUnCompressor unCompressor = benchmark.unCompressorSupplier.get();
+
+ byte[] compressed = new byte[0];
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < REPEAT_TIME; repeat++) {
+ compressed = compressor.compress(inputBytes);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / REPEAT_TIME;
+
+ byte[] restored = new byte[inputBytes.length];
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < REPEAT_TIME; repeat++) {
+ restored = new byte[inputBytes.length];
+ int uncompressedSize =
+ unCompressor.uncompress(compressed, 0, compressed.length, restored, 0);
+ Assert.assertEquals(inputBytes.length, uncompressedSize);
+ }
+ end = System.nanoTime();
+ long decodeTime = (end - start) / REPEAT_TIME;
+
+ Assert.assertArrayEquals(inputBytes, restored);
+ BenchmarkResult result =
+ new BenchmarkResult(
+ encodeTime,
+ decodeTime,
+ compressed.length,
+ compressed.length / (double) inputBytes.length);
+
+ String[] record = {
+ datasetName,
+ benchmark.algorithm,
+ String.valueOf(result.encodeTime),
+ String.valueOf(result.decodeTime),
+ String.valueOf(data.length),
+ String.valueOf(inputBytes.length),
+ String.valueOf(result.compressedSize),
+ String.valueOf(result.compressionRatio)
+ };
+ benchmark.writer.writeRecord(record);
+ }
+ }
+ } finally {
+ for (CompressionBenchmark benchmark : benchmarks) {
+ if (benchmark.writer != null) {
+ benchmark.writer.close();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java
new file mode 100644
index 0000000..094fbf3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ALPTest.java
@@ -0,0 +1,695 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class ALPTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int encode_pos,
+ int max_decimal,
+ byte[] encoded_result) {
+ int e = 0;
+ int f = 0;
+
+ int base = block_index * block_size;
+
+ e = 17;
+ f = 17 - max_decimal;
+
+ double[] testValues = new double[remainder];
+ for (int i = 0; i < testValues.length; i++) {
+ testValues[i] = data[base + (i * remainder / testValues.length)];
+ }
+
+ outer_loop: for (int i = 0; i < 19; i++) {
+ for (int j = i; j >= 0; j--) {
+ boolean valid = true;
+
+ for (double value : testValues) {
+ // double encodedValue = n * Math.pow(10, i) * Math.pow(10, -j);
+ // double encodedValue = n * power(10, i) / power(10, j);
+ long encodedValue = Math.round(value * power(10, i) / power(10, j));
+
+ double decodedValue = encodedValue / power(10, i) * power(10, j);
+ // if (Math.abs(n - decodedValue) > 0.00001) {
+ if (Math.abs(value - decodedValue) > Math.pow(10, -max_decimal)) {
+ valid = false;
+ break;
+ }
+ }
+
+ if (valid) {
+ e = i;
+ f = j;
+
+ break outer_loop;
+ }
+ }
+ }
+
+ intByte2Bytes(e, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ intByte2Bytes(f, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ long[] data_long = new long[remainder];
+
+ long min_data = Long.MAX_VALUE;
+ long max_data = Long.MIN_VALUE;
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] = Math.round(data[base + i] * power(10, e) / power(10, f));
+ if (data_long[i] < min_data) {
+ min_data = data_long[i];
+ }
+ if (data_long[i] > max_data) {
+ max_data = data_long[i];
+ }
+ }
+
+ long2Bytes(min_data, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] -= min_data;
+ }
+ long max_diff = max_data - min_data;
+ int bit_width = bitWidth(max_diff);
+
+ intByte2Bytes(bit_width, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(data_long, bit_width, encode_pos, encoded_result, remainder);
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, double[] data) {
+ int e = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int f = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ long min_data = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // System.out.println("min_data: " + min_data);
+
+ int bit_width = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ // System.out.println("bit_width: " + bit_width);
+
+ long[] data_long = new long[remainder];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bit_width, remainder, data_long);
+
+ for (int i = 0; i < remainder; i++) {
+ data_long[i] += min_data;
+ // data[block_index * block_size + i] = data_long[i] * Math.pow(10, f) *
+ // Math.pow(10, -e);
+ data[block_index * block_size + i] = data_long[i] * power(10, f) / power(10, e);
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, max_decimal, encoded_result);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ max_decimal, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static double[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ double[] data = new double[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static double power(int base, int exponent) {
+ double result = 1;
+ for (int i = 0; i < exponent; i++) {
+ result *= base;
+ }
+ return result;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "alp.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ double[] data1_arr = new double[data1.size()];
+ for (int i = 0; i < data1.size(); i++) {
+ data1_arr[i] = data1.get(i);
+ }
+
+ byte[] encoded_result = new byte[data1.size() * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data1_arr, block_size, max_decimal, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ double[] data2_arr_decoded = new double[data1_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ALP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void test1() {
+ double c = Math.pow(10, -17);
+
+ System.out.println(c);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java
new file mode 100644
index 0000000..16c395c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ASubcolumnAddDictionaryAndHuffmanTest.java
@@ -0,0 +1,1295 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+public class ASubcolumnAddDictionaryAndHuffmanTest {
+
+ private static final class HuffmanNode {
+ final int symbol; // -1 for internal node
+ final int freq;
+ final HuffmanNode left;
+ final HuffmanNode right;
+
+ private HuffmanNode(int symbol, int freq, HuffmanNode left, HuffmanNode right) {
+ this.symbol = symbol;
+ this.freq = freq;
+ this.left = left;
+ this.right = right;
+ }
+
+ boolean isLeaf() {
+ return left == null && right == null;
+ }
+ }
+
+ private static final class HuffmanCode {
+ final int code;
+ final int len;
+
+ private HuffmanCode(int code, int len) {
+ this.code = code;
+ this.len = len;
+ }
+ }
+
+ private static final class HuffmanBuildResult {
+ final int[] symbolsSortedAsc;
+ final byte[] codeLenBySymbolIndex; // aligned with symbolsSortedAsc
+ final HuffmanCode[] canonicalCodesBySymbolIndex; // aligned with symbolsSortedAsc
+ final int totalBits;
+
+ private HuffmanBuildResult(
+ int[] symbolsSortedAsc,
+ byte[] codeLenBySymbolIndex,
+ HuffmanCode[] canonicalCodesBySymbolIndex,
+ int totalBits) {
+ this.symbolsSortedAsc = symbolsSortedAsc;
+ this.codeLenBySymbolIndex = codeLenBySymbolIndex;
+ this.canonicalCodesBySymbolIndex = canonicalCodesBySymbolIndex;
+ this.totalBits = totalBits;
+ }
+ }
+
+ private static void writeBits(int value, int bitLen, byte[] out, int bitPos) {
+ // Write highest bits first.
+ for (int k = bitLen - 1; k >= 0; k--) {
+ boolean b = ((value >>> k) & 1) != 0;
+ boolToBytes(b, out, bitPos++);
+ }
+ }
+
+ private static int readBit(byte[] in, int bitPos) {
+ return bytesToBool(in, bitPos) ? 1 : 0;
+ }
+
+ private static void assignCodeLengths(HuffmanNode node, int depth, Map<Integer, Integer> outLen) {
+ if (node.isLeaf()) {
+ // Single-symbol edge case: assign length 1.
+ outLen.put(node.symbol, Math.max(1, depth));
+ return;
+ }
+ assignCodeLengths(node.left, depth + 1, outLen);
+ assignCodeLengths(node.right, depth + 1, outLen);
+ }
+
+ private static HuffmanBuildResult buildCanonicalHuffman(int[] values, int valueCount) {
+ // Build frequency table
+ Map<Integer, Integer> freq = new HashMap<>();
+ for (int i = 0; i < valueCount; i++) {
+ freq.merge(values[i], 1, Integer::sum);
+ }
+
+ int cardinality = freq.size();
+ int[] symbols = new int[cardinality];
+ int idx = 0;
+ for (int s : freq.keySet()) {
+ symbols[idx++] = s;
+ }
+ Arrays.sort(symbols);
+
+ // Build Huffman tree
+ PriorityQueue<HuffmanNode> pq = new PriorityQueue<>(
+ Comparator.<HuffmanNode>comparingInt(n -> n.freq).thenComparingInt(n -> n.symbol));
+ for (int s : symbols) {
+ pq.add(new HuffmanNode(s, freq.get(s), null, null));
+ }
+ while (pq.size() > 1) {
+ HuffmanNode a = pq.poll();
+ HuffmanNode b = pq.poll();
+ // Ensure deterministic structure by ordering children by (freq,symbol)
+ HuffmanNode left = a;
+ HuffmanNode right = b;
+ pq.add(new HuffmanNode(-1, a.freq + b.freq, left, right));
+ }
+ HuffmanNode root = pq.poll();
+
+ Map<Integer, Integer> lenBySymbol = new HashMap<>();
+ assignCodeLengths(root, 0, lenBySymbol);
+
+ // Prepare canonical order: by (len, symbol)
+ Integer[] order = new Integer[cardinality];
+ for (int i = 0; i < cardinality; i++) {
+ order[i] = symbols[i];
+ }
+ Arrays.sort(order, Comparator.<Integer>comparingInt(s -> lenBySymbol.get(s)).thenComparingInt(s -> s));
+
+ // Assign canonical codes
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ int code = 0;
+ int prevLen = lenBySymbol.get(order[0]);
+ codeBySymbol.put(order[0], new HuffmanCode(code, prevLen));
+ for (int i = 1; i < order.length; i++) {
+ int s = order[i];
+ int len = lenBySymbol.get(s);
+ code = (code + 1) << (len - prevLen);
+ codeBySymbol.put(s, new HuffmanCode(code, len));
+ prevLen = len;
+ }
+
+ // Align to symbolsSortedAsc for compact lookup
+ byte[] codeLens = new byte[cardinality];
+ HuffmanCode[] codes = new HuffmanCode[cardinality];
+ int totalBits = 0;
+ for (int i = 0; i < cardinality; i++) {
+ int s = symbols[i];
+ HuffmanCode hc = codeBySymbol.get(s);
+ codeLens[i] = (byte) hc.len;
+ codes[i] = hc;
+ totalBits += freq.get(s) * hc.len;
+ }
+
+ return new HuffmanBuildResult(symbols, codeLens, codes, totalBits);
+ }
+
+ private static Map<Integer, HuffmanCode> rebuildCanonicalCodes(int[] symbolsSortedAsc, byte[] codeLens) {
+ int cardinality = symbolsSortedAsc.length;
+ Integer[] order = new Integer[cardinality];
+ for (int i = 0; i < cardinality; i++) {
+ order[i] = symbolsSortedAsc[i];
+ }
+ Map<Integer, Integer> lenBySymbol = new HashMap<>();
+ for (int i = 0; i < cardinality; i++) {
+ lenBySymbol.put(symbolsSortedAsc[i], codeLens[i] & 0xFF);
+ }
+ Arrays.sort(order, Comparator.<Integer>comparingInt(s -> lenBySymbol.get(s)).thenComparingInt(s -> s));
+
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ int code = 0;
+ int prevLen = lenBySymbol.get(order[0]);
+ codeBySymbol.put(order[0], new HuffmanCode(code, prevLen));
+ for (int i = 1; i < order.length; i++) {
+ int s = order[i];
+ int len = lenBySymbol.get(s);
+ code = (code + 1) << (len - prevLen);
+ codeBySymbol.put(s, new HuffmanCode(code, len));
+ prevLen = len;
+ }
+ return codeBySymbol;
+ }
+
+ private static final class MutableTrieNode {
+ MutableTrieNode zero;
+ MutableTrieNode one;
+ int symbol = -1;
+ }
+
+ private static MutableTrieNode buildMutableDecodeTrie(Map<Integer, HuffmanCode> codeBySymbol) {
+ MutableTrieNode root = new MutableTrieNode();
+ for (Map.Entry<Integer, HuffmanCode> e : codeBySymbol.entrySet()) {
+ int sym = e.getKey();
+ HuffmanCode hc = e.getValue();
+ MutableTrieNode cur = root;
+ for (int k = hc.len - 1; k >= 0; k--) {
+ int b = (hc.code >>> k) & 1;
+ if (b == 0) {
+ if (cur.zero == null) {
+ cur.zero = new MutableTrieNode();
+ }
+ cur = cur.zero;
+ } else {
+ if (cur.one == null) {
+ cur.one = new MutableTrieNode();
+ }
+ cur = cur.one;
+ }
+ }
+ cur.symbol = sym;
+ }
+ return root;
+ }
+
+ public static int bitWidth(int value) {
+ if (value == 0)
+ return 1;
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int cost0 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+
+ int count = 1;
+
+ de_cost_single[i] = 1;
+
+ for (int j = 0; j < x_length; j++) {
+
+ // if (count * (1 + (int) Math.ceil(Math.log(x_length))) >= x_length) {
+ // rle_cost_single[i] = x_length + 1;
+ // break;
+ // }
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length + 2 * (1 + 1);
+ }
+
+ }
+
+ rle_cost_single[i] = count * (1 + (int) Math.ceil(Math.log(x_length)));
+
+ cost0 += Math.min(bpe_cost_single[i], Math.min(rle_cost_single[i], de_cost_single[i]));
+ }
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ // int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ // for (int i = 0; i < l; i++) {
+ // int maxValuePart = 0;
+ // for (int j = 0; j < x_length; j++) {
+ // subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][j] > maxValuePart) {
+ // maxValuePart = subcolumnList[i][j];
+ // }
+ // }
+ // bitWidthListList[i] = bitWidth(maxValuePart);
+ // }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+
+ // int bpCost = bpe_cost_single[i * beta] * beta;
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start - 1 >= i * beta && bpe_cost_single[beta_start - 1] == 0) {
+ beta_start--;
+ }
+
+ int bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
+ int rleCost = 0;
+
+ // int lowestBitIndex = 0;
+ // int currentLowestBit = subcolumnList[i][0] & 1;
+
+ // for (int j = 1; j < x_length; j++) {
+ // int lowestBit = subcolumnList[i][j] & 1; // 获取当前元素的最低位
+ // if (lowestBit != currentLowestBit) {
+ // lowestBitIndex++;
+ // currentLowestBit = lowestBit;
+ // }
+ // }
+
+ // if (bw * lowestBitIndex + bitWidthListList[i] * lowestBitIndex >= bpCost) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+ int currentNumber = (x[0] >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 1; j < x_length; j++) {
+ int currentNumber_j = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (currentNumber_j != currentNumber) {
+ index++;
+ currentNumber = currentNumber_j;
+ }
+ if (bw * index + bitWidth(x_length) * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidth(x_length) * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) * 2 / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ // uniqueValues.add(previous);
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ // if(currentNumber == 6){
+ // System.out.println("currentNumber == 6 && i==0");
+ // System.out.println(uniqueValues);
+ // }
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+ Set<Integer> uniqueValues = new HashSet<>();
+ Map<Integer, Integer> freq = new HashMap<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ freq.merge(currentNumber, 1, Integer::sum);
+ }
+ int cardinality = uniqueValues.size();
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ // Huffman as the 4th option for each subcolumn
+ // We store: [cardinality:2B][totalBits:4B][symbols(bitpacked)][codeLen(bytes)][bitstream]
+ int hufTotalBits;
+ int hufOverheadBits;
+ HuffmanBuildResult hufBuild = null;
+ try {
+ hufBuild = buildCanonicalHuffman(subcolumnList[i], list_length);
+ hufTotalBits = hufBuild.totalBits;
+ // overhead: 2B + 4B + symbols(bitWidthList[i] * cardinality) + codeLen(8 * cardinality)
+ hufOverheadBits = (2 + 4) * 8 + bitWidthList[i] * cardinality + 8 * cardinality;
+ } catch (Exception ex) {
+ // If anything goes wrong, just disable Huffman for this subcolumn
+ hufTotalBits = Integer.MAX_VALUE / 2;
+ hufOverheadBits = Integer.MAX_VALUE / 2;
+ }
+ int hufCostBits = (hufTotalBits >= Integer.MAX_VALUE / 4) ? Integer.MAX_VALUE : (hufTotalBits + hufOverheadBits);
+
+ if (cardinality < Math.pow(2, bitWidthList[i] - 1)) {
+ // test dictionary encoding
+ int dict_bit_width = bitWidth(cardinality);
+ int dicCost = dict_bit_width * list_length + cardinality * (bitWidthList[i] + dict_bit_width);
+ if (dicCost < rleCost && dicCost < bpCost) {
+ // if dictionary encoding
+ // int dict_bit_width = bitWidth(cardinality) ;
+ encodingType[i] = 2;
+
+ // System.out.println(uniqueValues);
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ // int[] dict_value_list = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ // dict_value_list[j] = j;
+ }
+ // System.out.println(valueToCode);
+ // System.out.println(list_length);
+ // System.out.println(beta[0]);
+ // System.out.println(Arrays.toString(subcolumnList[i]));
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+ // encode_pos = bitPacking(dict_value_list, dict_bit_width, encode_pos,
+ // encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+ }
+
+ // If Huffman wins, encode Huffman
+ if (hufCostBits < bpCost && hufCostBits < rleCost) {
+ encodingType[i] = 3;
+
+ // cardinality
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ // totalBits
+ int2Bytes(hufBuild.totalBits, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // symbols sorted asc
+ encode_pos = bitPacking(hufBuild.symbolsSortedAsc, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ // code lengths in bytes aligned with symbolsSortedAsc
+ for (int j = 0; j < cardinality; j++) {
+ encoded_result[encode_pos + j] = hufBuild.codeLenBySymbolIndex[j];
+ }
+ encode_pos += cardinality;
+
+ // build lookup from symbol->(code,len) by symbolsSortedAsc index
+ Map<Integer, HuffmanCode> codeBySymbol = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ codeBySymbol.put(hufBuild.symbolsSortedAsc[j], hufBuild.canonicalCodesBySymbolIndex[j]);
+ }
+
+ int bitPos = encode_pos * 8;
+ int bytesToWrite = (hufBuild.totalBits + 7) / 8;
+ Arrays.fill(encoded_result, encode_pos, encode_pos + bytesToWrite, (byte) 0);
+ for (int j = 0; j < list_length; j++) {
+ int sym = subcolumnList[i][j];
+ HuffmanCode hc = codeBySymbol.get(sym);
+ writeBits(hc.code, hc.len, encoded_result, bitPos);
+ bitPos += hc.len;
+ }
+ encode_pos += bytesToWrite;
+ continue;
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if (type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else if (type == 2) {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width,
+ // cardinality, dict_value_list);
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length,
+ subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ } else {
+ // Huffman
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ int totalBits = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] symbolsSortedAsc = new int[cardinality];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, symbolsSortedAsc);
+
+ byte[] codeLens = new byte[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ codeLens[j] = encoded_result[encode_pos + j];
+ }
+ encode_pos += cardinality;
+
+ Map<Integer, HuffmanCode> codeBySymbol = rebuildCanonicalCodes(symbolsSortedAsc, codeLens);
+ MutableTrieNode trie = buildMutableDecodeTrie(codeBySymbol);
+
+ int bitPos = encode_pos * 8;
+ int bitsRead = 0;
+ for (int j = 0; j < list_length; j++) {
+ MutableTrieNode cur = trie;
+ while (cur.symbol < 0) {
+ int b = readBit(encoded_result, bitPos++);
+ bitsRead++;
+ cur = (b == 0) ? cur.zero : cur.one;
+ }
+ subcolumnList[i][j] = cur.symbol;
+ }
+
+ // advance bytes by totalBits (trusted) rather than bitsRead (derived)
+ encode_pos += (totalBits + 7) / 8;
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_dictionary_huffman.csv";
+
+ // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 10;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ // if(! datasetName.equals("Stocks-UK")){
+ // continue;
+ // }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Dictionary)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java
new file mode 100644
index 0000000..b23b8f7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPLongTest.java
@@ -0,0 +1,678 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BPLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BPEncoder(long[] list, int encode_pos, byte[] encoded_result) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ encoded_result[encode_pos] = (byte) m;
+ encode_pos += 1;
+
+ encode_pos = bitPacking(list, m, encode_pos, encoded_result, list_length);
+
+ return encode_pos;
+ }
+
+ public static int BPDecoder(byte[] encoded_result, int encode_pos, long[] list) {
+ int list_length = list.length;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ long[] new_list = new long[list_length];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, m, list_length, new_list);
+
+ for (int i = 0; i < list_length; i++) {
+ list[i] = new_list[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ encode_pos = BPEncoder(data_delta, encode_pos,
+ encoded_result);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = BPDecoder(encoded_result, encode_pos,
+ block_data);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "bp_long.csv";
+ String outputPath = output_parent_dir + "bp_long2.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data1.size()];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java
new file mode 100644
index 0000000..ddf28aa
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQueryGroupTest.java
@@ -0,0 +1,126 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPQueryGroupTest {
+
+ public static int[] queryGroupMaxIndexByDecode(byte[] encodedResult, int windowSize) {
+ int[] decoded = BPTest.Decoder(encodedResult);
+ int groupCount = (decoded.length + windowSize - 1) / windowSize;
+ int[] result = new int[groupCount];
+ for (int g = 0; g < groupCount; g++) {
+ int start = g * windowSize;
+ int end = Math.min(decoded.length, start + windowSize);
+ int bestIndex = start;
+ int bestValue = decoded[start];
+ for (int i = start + 1; i < end; i++) {
+ if (decoded[i] > bestValue) {
+ bestValue = decoded[i];
+ bestIndex = i;
+ }
+ }
+ result[g] = bestIndex;
+ }
+ return result;
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "bp_query_group_max.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+ int windowSize = 30;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = BPTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] dataArr = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 8)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BPTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ int[] groupResult = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ groupResult = queryGroupMaxIndexByDecode(encodedResult, windowSize);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println("groupCount: " + (groupResult == null ? 0 : groupResult.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BP",
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java
new file mode 100644
index 0000000..753e49d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPQuerySortTest.java
@@ -0,0 +1,142 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPQuerySortTest {
+
+ public static int[] querySortSingleBlockByDecode(byte[] encodedResult, int targetBlockId) {
+ int[] decoded = BPTest.Decoder(encodedResult);
+ int blockSize = ((encodedResult[4] & 0xFF) << 24)
+ | ((encodedResult[5] & 0xFF) << 16)
+ | ((encodedResult[6] & 0xFF) << 8)
+ | (encodedResult[7] & 0xFF);
+ int dataLength = decoded.length;
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+ if (targetBlockId < 0 || targetBlockId >= totalBlocks) {
+ return new int[0];
+ }
+
+ int count = targetBlockId < numBlocks ? blockSize : remainder;
+ int start = targetBlockId * blockSize;
+ int[] indices = new int[count];
+ for (int i = 0; i < count; i++) {
+ indices[i] = start + i;
+ }
+
+ for (int i = 0; i < count - 1; i++) {
+ int min = i;
+ for (int j = i + 1; j < count; j++) {
+ int v1 = decoded[indices[j]];
+ int v2 = decoded[indices[min]];
+ if (v1 < v2 || (v1 == v2 && indices[j] < indices[min])) {
+ min = j;
+ }
+ }
+ int t = indices[i];
+ indices[i] = indices[min];
+ indices[min] = t;
+ }
+ return indices;
+ }
+
+ @Test
+ public void testDecodeSortSingleBlock() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "bp_query_sort_decode.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+ int targetBlockId = 0;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = BPTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] dataArr = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 8)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BPTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ int[] sortedIndices = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ sortedIndices = querySortSingleBlockByDecode(encodedResult, targetBlockId);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println("sortedCount: " + (sortedIndices == null ? 0 : sortedIndices.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BP",
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
index 36f5a35..b7552ad 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPTest.java
@@ -18,23 +18,43 @@
public class BPTest {
+ private static final int PACK_BIT_STEP = 4;
+ private static final int BIT_IO_STEP = 4;
+
public static int bitWidth(int value) {
return 32 - Integer.numberOfLeadingZeros(value);
}
+ private static void storeByteBits(byte[] result, int index, int byteVal) {
+ storeByteBits(result, index, byteVal, PACK_BIT_STEP);
+ }
+
+ private static void storeByteBits(byte[] result, int index, int byteVal, int step) {
+ int bitIndex = 8;
+ int remaining = 8;
+ while (remaining > 0) {
+ int bitsToWrite = Math.min(step, Math.min(bitIndex, remaining));
+ bitIndex -= bitsToWrite;
+ int mask = (1 << bitsToWrite) - 1;
+ int bits = (byteVal >> (remaining - bitsToWrite)) & mask;
+ result[index] &= (byte) ~(mask << bitIndex);
+ result[index] |= (byte) (bits << bitIndex);
+ remaining -= bitsToWrite;
+ }
+ }
+
public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
int cnt = pos & 0x07;
int index = pos >> 3;
while (width > 0) {
- int m = width + cnt >= 8 ? 8 - cnt : width;
+ int available = 8 - cnt;
+ int m = Math.min(BIT_IO_STEP, Math.min(available, width));
width -= m;
- int mask = 1 << (8 - cnt);
+ int mask = (1 << m) - 1;
+ int bits = (srcNum >> width) & mask;
+ int byteMask = mask << (8 - cnt - m);
+ result[index] = (byte) (result[index] & ~byteMask | (bits << (8 - cnt - m)));
cnt += m;
- byte y = (byte) (srcNum >>> width);
- y = (byte) (y << (8 - cnt));
- mask = ~(mask - (1 << (8 - cnt)));
- result[index] = (byte) (result[index] & mask | y);
- srcNum = srcNum & ~(-1 << width);
if (cnt == 8) {
index++;
cnt = 0;
@@ -66,16 +86,12 @@
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,22 +100,23 @@
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ int outByte = (buffer >>> ((3 - j) * 8)) & 0xFF;
+ if (j == 3) {
+ storeByteBits(encoded_result, encode_pos, outByte, 3);
+ } else {
+ storeByteBits(encoded_result, encode_pos, outByte);
+ }
encode_pos++;
bufIdx++;
if (bufIdx >= width) {
@@ -107,31 +124,22 @@
}
}
}
-
}
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -162,12 +170,10 @@
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -192,15 +198,10 @@
}
int m = bitWidth(maxValue);
- // System.out.println("m: " + m);
- // writeBits(encoded_result, startBitPosition, 8, m);
- // startBitPosition += 8;
encoded_result[encode_pos] = (byte) m;
encode_pos += 1;
- // bitPacking(list, encoded_result, startBitPosition, m, list_length);
- // startBitPosition += m * list_length;
encode_pos = bitPacking(list, m, encode_pos, encoded_result, list_length);
return encode_pos;
@@ -372,15 +373,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -402,24 +400,19 @@
}
@Test
- public void testSubcolumn() throws IOException {
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
String parent_dir = "D:/github/xjz17/subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "bp.csv";
int block_size = 1024;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 500;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -436,7 +429,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -458,10 +450,7 @@
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
}
inputStream.close();
int[] data2_arr = new int[data1.size()];
@@ -491,11 +480,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -528,133 +513,4 @@
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "bp.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BP",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java
new file mode 100644
index 0000000..6b1714d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateAppendTest.java
@@ -0,0 +1,125 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateAppendTest {
+
+ private static void int2Bytes(int value, int pos, byte[] out) {
+ out[pos] = (byte) (value >> 24);
+ out[pos + 1] = (byte) (value >> 16);
+ out[pos + 2] = (byte) (value >> 8);
+ out[pos + 3] = (byte) value;
+ }
+
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "bp_update_append.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Append-only Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+ for (File file : csvFiles) {
+ String datasetName = BPTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) continue;
+ int d = BPTest.getDecimalPrecision(fStr);
+ if (d > maxDecimal) maxDecimal = d;
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+ int mul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ int[] origin = new int[data.size()];
+ int maxValue = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (int) (data.get(i) * mul);
+ if (origin[i] > maxValue) maxValue = origin[i];
+ }
+ if (origin.length == 0) continue;
+
+ int[] appended = new int[origin.length + 1];
+ System.arraycopy(origin, 0, appended, 0, origin.length);
+ appended[origin.length] = (maxValue == Integer.MAX_VALUE) ? maxValue : maxValue + 1;
+
+ byte[] encodedResult = new byte[Math.max(16, appended.length * 8)];
+ int length = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) length = BPTest.Encoder(origin, blockSize, encodedResult);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) continue;
+ int tailStart = 8;
+ for (int i = 0; i < numBlocks; i++) tailStart = skipBlock(encodedResult, tailStart, blockSize);
+ int newRemainder = remainder + 1;
+ int updatedLength = length;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ int pos = tailStart;
+ if (newRemainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ int2Bytes(appended[base + i], pos, encodedResult);
+ pos += 4;
+ }
+ } else {
+ pos = BPTest.BlockEncoder(appended, numBlocks, blockSize, newRemainder, pos, encodedResult);
+ }
+ updatedLength = pos;
+ }
+ e = System.nanoTime();
+ long appendTime = (e - s) / repeatTime;
+
+ double ratio = length / (double) (Math.max(1, origin.length) * Long.BYTES);
+ writer.writeRecord(new String[] {datasetName, "BP", String.valueOf(encodeTime), String.valueOf(appendTime),
+ String.valueOf(origin.length), String.valueOf(remainder), String.valueOf(updatedLength), String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java
new file mode 100644
index 0000000..113b04e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateDeleteSmallerTest.java
@@ -0,0 +1,72 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateDeleteSmallerTest {
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_delete_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Delete Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ for (int i = 0; i < data.size(); i++) origin[i] = (int) (data.get(i) * mul);
+ if (origin.length <= 1) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] deleted = new int[origin.length - 1];
+ System.arraycopy(origin, 0, deleted, 0, deleted.length);
+ int tailStart = 8;
+ for (int i = 0; i < num; i++) {
+ tailStart += 4;
+ int[] tmp = new int[blockSize];
+ tailStart = BPTest.BPDecoder(encoded, tailStart, tmp);
+ }
+ int newRem = rem - 1;
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ if (newRem == 0) {
+ updatedLen = tailStart;
+ } else {
+ updatedLen = BPTest.BlockEncoder(deleted, num, blockSize, newRem, tailStart, encoded);
+ }
+ }
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java
new file mode 100644
index 0000000..8e2dea1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertLargerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateInsertLargerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_insert_larger.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int max = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); max = Math.max(max, origin[i]); }
+ if (origin.length == 0) continue;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = max == Integer.MAX_VALUE ? max : max + 1;
+ byte[] encoded = new byte[Math.max(16, inserted.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int newRem = rem + 1, updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(inserted, num, blockSize, newRem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java
new file mode 100644
index 0000000..e9bb605
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateInsertSmallerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateInsertSmallerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_insert_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int min = Integer.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); min = Math.min(min, origin[i]); }
+ if (origin.length == 0) continue;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = min == Integer.MIN_VALUE ? min : min - 1;
+ byte[] encoded = new byte[Math.max(16, inserted.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int newRem = rem + 1, updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(inserted, num, blockSize, newRem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java
new file mode 100644
index 0000000..66e5fc7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateLargerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateLargerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_larger.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int max = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); max = Math.max(max, origin[i]); }
+ if (origin.length == 0) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[num * blockSize + rem - 1] = max == Integer.MAX_VALUE ? max : max + 1;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(updated, num, blockSize, rem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java
new file mode 100644
index 0000000..ea9f4d7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BPUpdateSmallerTest.java
@@ -0,0 +1,68 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class BPUpdateSmallerTest {
+ private static int skipBlock(byte[] encodedResult, int encodePos, int rowCount) {
+ encodePos += 4;
+ int[] tmp = new int[rowCount];
+ return BPTest.BPDecoder(encodedResult, encodePos, tmp);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/update/bp_update_smaller.csv";
+ int blockSize = 512, repeatTime = 200;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(new String[]{"Dataset","Encoding Algorithm","Encoding Time","Insert Time with Sub-column","Points","Remaining Points","Compressed Size","Compression Ratio"});
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) { writer.close(); return; }
+ for (File file : csvFiles) {
+ String name = BPTest.extractFileName(file.toString());
+ if (name.equals("POI-lon") || name.equals("POI-lat")) continue;
+ InputStream is = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(is, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int dec = 0;
+ while (loader.readRecord()) { String s = loader.getValues()[0]; if (s.isEmpty()) continue; dec = Math.max(dec, BPTest.getDecimalPrecision(s)); data.add(Float.valueOf(s)); }
+ is.close();
+ int mul = (int) Math.pow(10, Math.min(dec, 8));
+ int[] origin = new int[data.size()];
+ int min = Integer.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) { origin[i] = (int) (data.get(i) * mul); min = Math.min(min, origin[i]); }
+ if (origin.length == 0) continue;
+ byte[] encoded = new byte[Math.max(16, origin.length * 8)];
+ int len = 0;
+ long s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) len = BPTest.Encoder(origin, blockSize, encoded);
+ long e = System.nanoTime();
+ long encodeTime = (e - s) / repeatTime;
+ int num = origin.length / blockSize, rem = origin.length % blockSize;
+ if (rem <= 0) continue;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[num * blockSize + rem - 1] = min == Integer.MIN_VALUE ? min : min - 1;
+ int tail = 8; for (int i=0;i<num;i++) tail = skipBlock(encoded, tail, blockSize);
+ int updatedLen = len;
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) updatedLen = BPTest.BlockEncoder(updated, num, blockSize, rem, tail, encoded);
+ e = System.nanoTime();
+ long t = (e - s) / repeatTime;
+ double ratio = len / (double)(Math.max(1,origin.length)*Long.BYTES);
+ writer.writeRecord(new String[]{name,"BP",String.valueOf(encodeTime),String.valueOf(t),String.valueOf(origin.length),String.valueOf(rem),String.valueOf(updatedLen),String.valueOf(ratio)});
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java
new file mode 100644
index 0000000..3cafe8a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDouble2Test.java
@@ -0,0 +1,553 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BUFFDouble2Test {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ // 逐位写入整数到字节数组
+ public static void writeBits(int srcNum, byte[] result, int bitPos, int width) {
+ for (int i = 0; i < width; i++) {
+ int bit = (srcNum >> (width - 1 - i)) & 1;
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8); // 从最高位开始
+
+ if (bit == 1) {
+ result[byteIndex] |= (1 << bitOffset);
+ } else {
+ result[byteIndex] &= ~(1 << bitOffset);
+ }
+ bitPos++;
+ }
+ }
+
+ // 从字节数组逐位读取整数
+ public static int readBits(byte[] result, int bitPos, int width) {
+ int ret = 0;
+ for (int i = 0; i < width; i++) {
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8);
+ int bit = (result[byteIndex] >> bitOffset) & 1;
+ ret = (ret << 1) | bit;
+ bitPos++;
+ }
+ return ret;
+ }
+
+ // 逐位写入长整数到字节数组
+ public static void writeBits(long srcNum, byte[] result, int bitPos, int width) {
+ for (int i = 0; i < width; i++) {
+ long bit = (srcNum >> (width - 1 - i)) & 1L;
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8); // 从最高位开始
+
+ if (bit == 1) {
+ result[byteIndex] |= (1 << bitOffset);
+ } else {
+ result[byteIndex] &= ~(1 << bitOffset);
+ }
+ bitPos++;
+ }
+ }
+
+ // 从字节数组逐位读取长整数
+ public static long readBitsLong(byte[] result, int bitPos, int width) {
+ long ret = 0;
+ for (int i = 0; i < width; i++) {
+ int byteIndex = bitPos / 8;
+ int bitOffset = 7 - (bitPos % 8);
+ long bit = (result[byteIndex] >> bitOffset) & 1L;
+ ret = (ret << 1) | bit;
+ bitPos++;
+ }
+ return ret;
+ }
+
+ // 逐位打包整数数组
+ public static int bitPacking(int[] numbers, int bit_width, int startBitPos,
+ byte[] encoded_result, int num_values) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ writeBits(numbers[i], encoded_result, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位打包长整数数组
+ public static int bitPacking(long[] numbers, int bit_width, int startBitPos,
+ byte[] encoded_result, int num_values) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ writeBits(numbers[i], encoded_result, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位解包整数数组
+ public static int decodeBitPacking(
+ byte[] encoded, int startBitPos, int bit_width, int num_values, int[] result_list) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ result_list[i] = readBits(encoded, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 逐位解包长整数数组
+ public static int decodeBitPacking(
+ byte[] encoded, int startBitPos, int bit_width, int num_values, long[] result_list) {
+ int currentBitPos = startBitPos;
+
+ for (int i = 0; i < num_values; i++) {
+ result_list[i] = readBitsLong(encoded, currentBitPos, bit_width);
+ currentBitPos += bit_width;
+ }
+
+ return (currentBitPos + 7) / 8; // 返回字节位置
+ }
+
+ // 辅助函数：将整数写入字节数组（字节对齐）
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ // 辅助函数：将长整数写入字节数组（字节对齐）
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ // 从字节数组读取整数
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ // 从字节数组读取长整数
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int max_decimal,
+ int encode_pos, byte[] encoded_result) {
+
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
+
+ long min_integer_part = Long.MAX_VALUE;
+ long max_integer_part = Long.MIN_VALUE;
+
+ // 提取每个双精度值的符号、整数部分和小数部分
+ for (int i = 0; i < remainder; i++) {
+ double value = data[block_index * block_size + i];
+
+ // 符号位
+ if (value < 0) {
+ sign_bits[i] = 1;
+ }
+
+ // 整数部分
+ long currentInt = (long) Math.abs(value);
+ integer_parts[i] = currentInt;
+
+ // 更新最小和最大整数部分
+ if (currentInt < min_integer_part) {
+ min_integer_part = currentInt;
+ }
+ if (currentInt > max_integer_part) {
+ max_integer_part = currentInt;
+ }
+
+ // 提取小数部分
+ long bits = Double.doubleToLongBits(value);
+ long exponent = (bits >> 52) & 0x7FF;
+ long mantissa = bits & (long) ((1L << 52) - 1);
+ long actualExponent = exponent - 1023;
+
+ if (actualExponent >= 0) {
+ long mask = (1L << (52 - actualExponent)) - 1;
+ mantissa &= mask;
+ } else {
+ mantissa += 1L << 52;
+ }
+
+ long shift = 52 - actualExponent - bits_needed[max_decimal];
+ if (shift < 0) {
+ mantissa <<= -shift;
+ } else {
+ mantissa >>= shift;
+ }
+
+ if (exponent == 0) {
+ mantissa = 0;
+ }
+
+ decimal_parts[i] = mantissa;
+ }
+
+ // 写入最小整数部分（8字节）
+ long2Bytes(min_integer_part, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ // 计算整数部分所需的位数并写入
+ int bw = bitWidth(max_integer_part - min_integer_part);
+ encoded_result[encode_pos] = (byte) bw;
+ encode_pos += 1;
+
+ // 对整数部分进行差分编码
+ for (int i = 0; i < remainder; i++) {
+ integer_parts[i] -= min_integer_part;
+ }
+
+ // 计算总位宽
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ // 为每个值分配一个长整数数组来存储组合位
+ long[] combinedValues = new long[remainder];
+
+ // 将符号位、整数部分和小数部分组合成一个长整数
+ for (int i = 0; i < remainder; i++) {
+ long combined = (sign_bits[i] << (bw + bits_needed[max_decimal]))
+ | (integer_parts[i] << bits_needed[max_decimal]) | decimal_parts[i];
+ combinedValues[i] = combined;
+ }
+
+ // 使用逐位打包方式编码组合值
+ int bitPos = encode_pos * 8; // 转换为位位置
+ encode_pos = bitPacking(combinedValues, totalBitWidth, bitPos, encoded_result, remainder);
+
+ return encode_pos;
+ }
+
+ // 块解码器
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int max_decimal, int encode_pos, double[] data) {
+
+ // 读取最小整数部分
+ long min_integer_part = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // 读取整数部分位宽
+ int bw = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 计算总位宽
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ // 解码组合值
+ long[] combinedValues = new long[remainder];
+ int bitPos = encode_pos * 8; // 转换为位位置
+ encode_pos = decodeBitPacking(encoded_result, bitPos, totalBitWidth, remainder, combinedValues);
+
+ // 解码每个值
+ for (int i = 0; i < remainder; i++) {
+ long combined = combinedValues[i];
+
+ // 提取符号位、整数部分和小数部分
+ long sign_bit = (combined >> (bw + bits_needed[max_decimal])) & 1L;
+ long integer_part = (combined >> bits_needed[max_decimal]) & ((1L << bw) - 1);
+ long decimal_part = combined & ((1L << bits_needed[max_decimal]) - 1);
+
+ // 恢复原始整数部分
+ integer_part += min_integer_part;
+
+ // 将小数部分转换为double
+ double decimal = decimal_part;
+ for (int j = 0; j < bits_needed[max_decimal]; j++) {
+ decimal /= 2;
+ }
+
+ // 组合成最终的双精度值
+ double value = integer_part + decimal;
+ value = sign_bit == 1 ? -value : value;
+ data[block_index * block_size + i] = value;
+ }
+
+ return encode_pos;
+ }
+
+ // 主编码器
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ // 写入数据长度（4字节）
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ // 写入块大小（4字节）
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ // 写入最大小数位数（1字节）
+ encoded_result[8] = (byte) max_decimal;
+ encode_pos += 1;
+
+ // 计算块数
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 编码完整块
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, max_decimal, encode_pos, encoded_result);
+ }
+
+ // 编码剩余部分
+ if (remainder > 0) {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ // 主解码器
+ public static double[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ // 读取数据长度
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // 读取块大小
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // 读取最大小数位数
+ int max_decimal = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 计算块数
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 分配结果数组
+ double[] data = new double[data_length];
+
+ // 解码完整块
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, max_decimal, encode_pos, data);
+ }
+
+ // 解码剩余部分
+ if (remainder > 0) {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "buff_long0.csv";
+ String outputPath = output_parent_dir + "buff_long_repeat50.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+ double[] data2_arr = new double[data1.size()];
+
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = data1.get(i);
+ }
+
+ System.out.println(max_decimal);
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, max_decimal, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ double[] data2_arr_decoded = new double[data1.size()];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java
new file mode 100644
index 0000000..1f8ea14
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFDoubleTest.java
@@ -0,0 +1,714 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BUFFDoubleTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
+
+ public static int BlockEncoder(double[] data, int block_index, int block_size, int remainder, int max_decimal,
+ int encode_pos, byte[] encoded_result) {
+
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
+
+ long min_integer_part = Long.MAX_VALUE;
+ long max_integer_part = Long.MIN_VALUE;
+
+ for (int i = 0; i < remainder; i++) {
+ double value = data[block_index * block_size + i];
+
+ if (value < 0) {
+ sign_bits[i] = 1;
+ }
+
+ long currentInt = (long) Math.abs(value);
+ integer_parts[i] = currentInt;
+
+ if (currentInt < min_integer_part) {
+ min_integer_part = currentInt;
+ }
+
+ if (currentInt > max_integer_part) {
+ max_integer_part = currentInt;
+ }
+
+ long bits = Double.doubleToLongBits(value);
+
+ long exponent = (bits >> 52) & 0x7FF;
+ long mantissa = bits & (long) ((1L << 52) - 1);
+
+ long actualExponent = exponent - 1023;
+
+ if (actualExponent >= 0) {
+ long mask = (1L << (52 - actualExponent)) - 1;
+ mantissa &= mask;
+ } else {
+ mantissa += 1L << 52;
+ }
+
+ long shift = 52 - actualExponent - bits_needed[max_decimal];
+
+ if (shift < 0) {
+ mantissa <<= -shift;
+ } else {
+ mantissa >>= shift;
+ }
+
+ if (exponent == 0) {
+ mantissa = 0;
+ }
+
+ decimal_parts[i] = mantissa;
+ }
+
+ long2Bytes(min_integer_part, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_integer_part - min_integer_part);
+
+ encoded_result[encode_pos] = (byte) bw;
+ encode_pos += 1;
+
+ for (int i = 0; i < remainder; i++) {
+ integer_parts[i] -= min_integer_part;
+ }
+
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ int intArrayCount = (totalBitWidth + 7) / 8;
+
+ long[][] combinedArrays = new long[intArrayCount][remainder];
+
+ for (int i = 0; i < intArrayCount; i++) {
+ for (int j = 0; j < remainder; j++) {
+ long combined = (sign_bits[j] << (bw + bits_needed[max_decimal]))
+ | (integer_parts[j] << bits_needed[max_decimal]) | decimal_parts[j];
+ combinedArrays[i][j] = ((combined >> (i * 8)) & 0xFF);
+ }
+ }
+
+ for (int i = 0; i < intArrayCount; i++) {
+ int currentBitWidth = Math.min(8, totalBitWidth - i * 8);
+ encode_pos = bitPacking(combinedArrays[i], currentBitWidth, encode_pos, encoded_result, remainder);
+ }
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int max_decimal, int encode_pos, double[] data) {
+
+ long[] sign_bits = new long[remainder];
+ long[] integer_parts = new long[remainder];
+ long[] decimal_parts = new long[remainder];
+
+ long min_integer_part = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int totalBitWidth = 1 + bw + bits_needed[max_decimal];
+
+ int intArrayCount = (totalBitWidth + 7) / 8;
+
+ long[][] combinedArrays = new long[intArrayCount][remainder];
+
+ long[] combined = new long[remainder];
+
+ for (int i = 0; i < intArrayCount; i++) {
+ int currentBitWidth = Math.min(8, totalBitWidth - i * 8);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, currentBitWidth, remainder, combinedArrays[i]);
+ for (int j = 0; j < remainder; j++) {
+ combined[j] |= (combinedArrays[i][j]) << (i * 8);
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ sign_bits[i] = ((combined[i] >> (bw + bits_needed[max_decimal])) & 1);
+ integer_parts[i] = ((combined[i] >> bits_needed[max_decimal]) & ((1 << bw) - 1));
+ integer_parts[i] += min_integer_part;
+ decimal_parts[i] = (combined[i] & ((1 << bits_needed[max_decimal]) - 1));
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ double decimal = decimal_parts[i];
+ for (int j = 0; j < bits_needed[max_decimal]; j++) {
+ decimal /= 2;
+ }
+ double value = (integer_parts[i] + decimal);
+ value = sign_bits[i] == 1 ? -value : value;
+ data[block_index * block_size + i] = value;
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(double[] data, int block_size, int max_decimal, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ encoded_result[8] = (byte) max_decimal;
+ encode_pos += 1;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, max_decimal, encode_pos, encoded_result);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static double[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int max_decimal = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ double[] data = new double[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, max_decimal, encode_pos, data);
+ }
+
+ if (remainder > 0) {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
+ String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+ String input_parent_dir = parent_dir + "ElfTestData_camel/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "buff_long0.csv";
+ String outputPath = output_parent_dir + "buff_long2_repeat50.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+ double[] data2_arr = new double[data1.size()];
+
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = data1.get(i);
+ }
+
+ System.out.println(max_decimal);
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, max_decimal, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ double[] data2_arr_decoded = new double[data1.size()];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
index 57ad31d..908eec8 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BUFFTest.java
@@ -66,16 +66,12 @@
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values[valueIdx] << (32 - leftBit));
leftSize -= leftBit;
@@ -84,20 +80,16 @@
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values[valueIdx] << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
+
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values[valueIdx] >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -113,25 +105,17 @@
public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
- // result_list.add((int) (buffer >>> (totalBits - width)));
result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
valueIdx++;
totalBits -= width;
@@ -162,12 +146,10 @@
public static int decodeBitPacking(
byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
- // ArrayList<Integer> result_list = new ArrayList<>();
- // int[] result_list = new int[num_values];
int block_num = num_values / 8;
int remainder = num_values % 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
decode_pos += bit_width;
}
@@ -182,7 +164,9 @@
return (decode_pos + 7) / 8;
}
- public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25 };
+ public static int[] bits_needed = { 0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35,
+ 38, 41, 45, 48, 51, 55, 58
+ };
public static int BlockEncoder(float[] data, int block_index, int block_size, int remainder, int max_decimal,
int encode_pos, byte[] encoded_result) {
@@ -194,11 +178,6 @@
int min_integer_part = Integer.MAX_VALUE;
int max_integer_part = Integer.MIN_VALUE;
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
-
for (int i = 0; i < remainder; i++) {
float value = data[block_index * block_size + i];
@@ -253,9 +232,6 @@
encoded_result[encode_pos + 3] = (byte) min_integer_part;
encode_pos += 4;
- // System.out.println("min_integer_part: " + min_integer_part);
- // System.out.println("max_integer_part: " + max_integer_part);
-
int bw = bitWidth(max_integer_part - min_integer_part);
encoded_result[encode_pos] = (byte) bw;
@@ -265,17 +241,6 @@
integer_parts[i] -= min_integer_part;
}
- // int[] combined = new int[remainder];
- // for (int i = 0; i < remainder; i++) {
- // combined[i] = (sign_bits[i] << (bw + bits_needed[max_decimal])) |
- // (integer_parts[i] << bits_needed[max_decimal]) | decimal_parts[i];
- // }
-
- // int totalBitWidth = 1 + bw + bits_needed[max_decimal];
-
- // encode_pos = bitPacking(combined, totalBitWidth, encode_pos, encoded_result,
- // remainder);
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
@@ -311,24 +276,10 @@
((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
encode_pos += 4;
- // System.out.println("min_integer_part: " + min_integer_part);
int bw = encoded_result[encode_pos];
encode_pos += 1;
- // int[] combined = new int[remainder];
-
- // encode_pos = decodeBitPacking(encoded_result, encode_pos, 1 + bw +
- // bits_needed[max_decimal], remainder, combined);
-
- // for (int i = 0; i < remainder; i++) {
- // int value = combined[i];
- // sign_bits[i] = (value >> (bw + bits_needed[max_decimal])) & 1;
- // integer_parts[i] = (value >> bits_needed[max_decimal]) & ((1 << bw) - 1);
- // integer_parts[i] += min_integer_part;
- // decimal_parts[i] = value & ((1 << bits_needed[max_decimal]) - 1);
- // }
-
int totalBitWidth = 1 + bw + bits_needed[max_decimal];
int intArrayCount = (totalBitWidth + 7) / 8;
@@ -362,11 +313,6 @@
data[block_index * block_size + i] = value;
}
- // for (int i = 0; i < remainder; i++) {
- // System.out.print(data[block_index * block_size + i] + " ");
- // }
- // System.out.println();
-
return encode_pos;
}
@@ -401,21 +347,6 @@
encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, max_decimal, encode_pos, encoded_result);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // int value = data[num_blocks * block_size + i];
- // encoded_result[encode_pos] = (byte) (value >> 24);
- // encoded_result[encode_pos + 1] = (byte) (value >> 16);
- // encoded_result[encode_pos + 2] = (byte) (value >> 8);
- // encoded_result[encode_pos + 3] = (byte) value;
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockEncoder(data, num_blocks, block_size, remainder,
- // max_decimal, encode_pos,
- // encoded_result);
- // }
-
return encode_pos;
}
@@ -448,33 +379,16 @@
encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, max_decimal, encode_pos, data);
}
- // if (remainder <= 3) {
- // for (int i = 0; i < remainder; i++) {
- // data[num_blocks * block_size + i] = ((encoded_result[encode_pos] & 0xFF) <<
- // 24) |
- // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos +
- // 3] & 0xFF);
- // encode_pos += 4;
- // }
- // } else {
- // encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
- // encode_pos, data);
- // }
-
return data;
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -496,13 +410,12 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "buff.csv";
@@ -510,11 +423,6 @@
int repeatTime = 100;
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -530,7 +438,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -553,18 +460,20 @@
max_decimal = cur_decimal;
data1.add(Float.valueOf(f_str));
}
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
inputStream.close();
- // int[] data2_arr = new int[data1.size()];
float[] data2_arr = new float[data1.size()];
- // int max_mul = (int) Math.pow(10, max_decimal);
for (int i = 0; i < data1.size(); i++) {
- // data2_arr[i] = (int) (data1.get(i) * max_mul);
data2_arr[i] = data1.get(i);
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -584,11 +493,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -621,135 +526,4 @@
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "buff.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- int max_decimal = 0;
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- int cur_decimal = getDecimalPrecision(loader.getValues()[1]);
- max_decimal = Math.max(max_decimal, cur_decimal);
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- float[] data2_arr = new float[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), max_decimal, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- float[] data2_arr_decoded = new float[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "BUFF",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java
new file mode 100644
index 0000000..820a27f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BitWeaving.java
@@ -0,0 +1,182 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.BitSet;
+import java.util.Random;
+import java.util.Arrays;
+
+public class BitWeaving {
+ private final long[][] bitPlanes;
+ private final int bitWidth;
+ private final int rowCount;
+ private final int wordsPerPlane;
+
+ public BitWeaving(int[] values) {
+ if (values == null) throw new IllegalArgumentException("values == null");
+ this.rowCount = values.length;
+ if (rowCount == 0) {
+ this.bitWidth = 1;
+ this.wordsPerPlane = 0;
+ this.bitPlanes = new long[bitWidth][0];
+ return;
+ }
+ int maxV = 0;
+ for (int v : values) {
+ if (v < 0) throw new IllegalArgumentException("This simple implementation expects non-negative integers. v=" + v);
+ if (v > maxV) maxV = v;
+ }
+ this.bitWidth = Math.max(1, 32 - Integer.numberOfLeadingZeros(maxV));
+ this.wordsPerPlane = (rowCount + 63) / 64;
+ this.bitPlanes = new long[bitWidth][wordsPerPlane];
+
+ for (int bit = 0; bit < bitWidth; ++bit) {
+ long[] plane = bitPlanes[bit];
+ for (int row = 0; row < rowCount; ++row) {
+ if (((values[row] >> bit) & 1) != 0) {
+ int wordIdx = row >>> 6;
+ int bitInWord = row & 63;
+ plane[wordIdx] |= (1L << bitInWord);
+ }
+ }
+ }
+ }
+
+ public int rowCount() { return rowCount; }
+
+ private static void maskToBitSet(long mask, int baseIndex, int rowCount, BitSet out) {
+ int limit = Math.min(64, Math.max(0, rowCount - baseIndex));
+ for (int b = 0; b < limit; ++b) {
+ if ((mask & (1L << b)) != 0L) out.set(baseIndex + b);
+ }
+ }
+
+ public BitSet lessThanToConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ long lt = 0L;
+ long gt = 0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ lt |= (eq & ~planeWord);
+ eq &= planeWord;
+ } else {
+ gt |= (eq & planeWord);
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ lt &= validMask;
+ maskToBitSet(lt, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public BitSet equalToConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ eq &= planeWord;
+ } else {
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ eq &= validMask;
+ maskToBitSet(eq, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public BitSet greaterThanConst(int K) {
+ BitSet result = new BitSet(rowCount);
+ if (rowCount == 0) return result;
+ for (int w = 0; w < wordsPerPlane; ++w) {
+ long eq = ~0L;
+ long gt = 0L;
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ long planeWord = bitPlanes[bit][w];
+ if (((K >> bit) & 1) == 1) {
+ eq &= planeWord;
+ } else {
+ gt |= (eq & planeWord);
+ eq &= ~planeWord;
+ }
+ }
+ int baseIndex = w << 6;
+ int remain = Math.max(0, rowCount - baseIndex);
+ long validMask = (remain >= 64) ? ~0L : ((1L << remain) - 1L);
+ gt &= validMask;
+ maskToBitSet(gt, baseIndex, rowCount, result);
+ }
+ return result;
+ }
+
+ public void dumpPlanes() {
+ System.out.println("bitWidth=" + bitWidth + ", rowCount=" + rowCount + ", wordsPerPlane=" + wordsPerPlane);
+ for (int bit = bitWidth - 1; bit >= 0; --bit) {
+ System.out.print("bit " + bit + " : ");
+ for (int r = 0; r < rowCount; ++r) {
+ int wi = r >>> 6;
+ int bi = r & 63;
+ long word = bitPlanes[bit][wi];
+ System.out.print(((word >>> bi) & 1L));
+ }
+ System.out.println();
+ }
+ }
+
+ public static void main(String[] args) {
+ final int N = 130;
+ final int MAXV = 31;
+ int[] values = new int[N];
+ Random rnd = new Random(12345);
+ for (int i = 0; i < N; ++i) values[i] = rnd.nextInt(MAXV + 1);
+
+ BitWeaving bw = new BitWeaving(values);
+ System.out.println("Generated " + N + " random values (max " + MAXV + "). bitWidth used = " + bw.bitWidth);
+ for (int K = 0; K <= MAXV; ++K) {
+ BitSet lt = bw.lessThanToConst(K);
+ BitSet eq = bw.equalToConst(K);
+ BitSet gt = bw.greaterThanConst(K);
+
+ BitSet expLt = new BitSet(N);
+ BitSet expEq = new BitSet(N);
+ BitSet expGt = new BitSet(N);
+ for (int i = 0; i < N; ++i) {
+ if (values[i] < K) expLt.set(i);
+ else if (values[i] == K) expEq.set(i);
+ else expGt.set(i);
+ }
+
+ if (!lt.equals(expLt) || !eq.equals(expEq) || !gt.equals(expGt)) {
+ System.err.println("Mismatch for K=" + K);
+ System.err.println("values: " + Arrays.toString(values));
+ System.err.println("lt result: " + lt);
+ System.err.println("expected lt: " + expLt);
+ System.err.println("eq result: " + eq);
+ System.err.println("expected eq: " + expEq);
+ System.err.println("gt result: " + gt);
+ System.err.println("expected gt: " + expGt);
+ System.exit(2);
+ }
+ }
+ int Kdemo = 10;
+ BitSet ltd = bw.lessThanToConst(Kdemo);
+ BitSet eqd = bw.equalToConst(Kdemo);
+ BitSet gtd = bw.greaterThanConst(Kdemo);
+ for (int i = 0; i < Math.min(32, N); ++i) {
+ System.out.printf("%3d: %2d <%b =%b >%b\n", i, values[i], ltd.get(i), eqd.get(i), gtd.get(i));
+ }
+ }
+}
+
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java
new file mode 100644
index 0000000..c30e4ef
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffOfficialRatioTest.java
@@ -0,0 +1,321 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class BuffOfficialRatioTest {
+
+ private static final long EXP_MASK = 0x7FF0000000000000L;
+ private static final long FIRST_ONE = 0x8000000000000000L;
+ private static final double TIME_FACTOR_VS_BPE = 1.05;
+ private static final int[] PRECISION_BITS =
+ {0, 5, 8, 11, 15, 18, 21, 25, 28, 31, 35, 38, 50, 10, 10, 10};
+
+ private static int decimalPrecision(String value) {
+ int dot = value.indexOf('.');
+ return dot < 0 ? 0 : value.length() - dot - 1;
+ }
+
+ private static String extractFileName(String path) {
+ String name = new File(path).getName();
+ int dot = name.lastIndexOf('.');
+ return dot <= 0 ? name : name.substring(0, dot);
+ }
+
+ private static double precisionBound(int precision) {
+ if (precision <= 0) {
+ return 0.49;
+ }
+ StringBuilder builder = new StringBuilder("0.");
+ for (int i = 0; i < precision; i++) {
+ builder.append('0');
+ }
+ builder.append("49");
+ return Double.parseDouble(builder.toString());
+ }
+
+ private static int precisionExponent(double precision) {
+ long bits = Double.doubleToRawLongBits(precision);
+ return (int) ((bits & EXP_MASK) >>> 52) - 1023;
+ }
+
+ private static long fetchFixedAligned(double value, int precisionExp, int decimalLength) {
+ long bits = Double.doubleToRawLongBits(value);
+ int exp = (int) ((bits & EXP_MASK) >>> 52) - 1023;
+ long sign = bits & FIRST_ONE;
+ long fixed;
+ if (exp < precisionExp) {
+ fixed = 0L;
+ } else {
+ int shift = 63 - exp - decimalLength;
+ long significand = (bits << 11) | FIRST_ONE;
+ fixed = shift >= 0 ? significand >>> shift : significand << -shift;
+ if (sign != 0) {
+ fixed = ~(fixed - 1);
+ }
+ }
+ return fixed;
+ }
+
+ private static byte flip(byte value) {
+ return (byte) (value ^ 0x80);
+ }
+
+ private static int putIntLE(int value, byte[] out, int pos) {
+ out[pos++] = (byte) value;
+ out[pos++] = (byte) (value >>> 8);
+ out[pos++] = (byte) (value >>> 16);
+ out[pos++] = (byte) (value >>> 24);
+ return pos;
+ }
+
+ private static int getIntLE(byte[] in, int pos) {
+ return (in[pos] & 0xFF)
+ | ((in[pos + 1] & 0xFF) << 8)
+ | ((in[pos + 2] & 0xFF) << 16)
+ | ((in[pos + 3] & 0xFF) << 24);
+ }
+
+ private static Map<String, Double> loadScaledBpeTimes(String path, String timeColumn)
+ throws IOException {
+ Map<String, Double> times = new HashMap<>();
+ File file = new File(path);
+ if (!file.exists()) {
+ return times;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader reader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ if (!reader.readHeaders()) {
+ reader.close();
+ inputStream.close();
+ return times;
+ }
+
+ while (reader.readRecord()) {
+ String dataset = reader.get("Dataset");
+ long time = Long.parseLong(reader.get(timeColumn));
+ long points = Long.parseLong(reader.get("Points"));
+ times.put(dataset, (time / (double) points) * TIME_FACTOR_VS_BPE);
+ }
+ reader.close();
+ inputStream.close();
+ return times;
+ }
+
+ private static int putLongLE(long value, byte[] out, int pos) {
+ pos = putIntLE((int) value, out, pos);
+ return putIntLE((int) (value >>> 32), out, pos);
+ }
+
+ private static long getLongLE(byte[] in, int pos) {
+ long low = getIntLE(in, pos) & 0xFFFFFFFFL;
+ long high = getIntLE(in, pos + 4) & 0xFFFFFFFFL;
+ return low | (high << 32);
+ }
+
+ private static byte[] encode(double[] values, int precision) {
+ if (values.length == 0) {
+ return new byte[20];
+ }
+ int decLen = PRECISION_BITS[Math.min(Math.max(precision, 0), PRECISION_BITS.length - 1)];
+ int precisionExp = precisionExponent(precisionBound(precision));
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ long[] fixedValues = new long[values.length];
+
+ for (int i = 0; i < values.length; i++) {
+ long fixed = fetchFixedAligned(values[i], precisionExp, decLen);
+ fixedValues[i] = fixed;
+ if (fixed < min) {
+ min = fixed;
+ }
+ if (fixed > max) {
+ max = fixed;
+ }
+ }
+
+ long delta = max - min;
+ int fixedLen = delta == 0 ? 0 : 64 - Long.numberOfLeadingZeros(delta - 1);
+ int bytesPerValue = Math.max(1, (fixedLen + 7) / 8);
+ byte[] encoded = new byte[20 + values.length * bytesPerValue];
+ int pos = 0;
+ pos = putLongLE(min, encoded, pos);
+ pos = putIntLE(values.length, encoded, pos);
+ pos = putIntLE(Math.max(0, fixedLen - decLen), encoded, pos);
+ pos = putIntLE(decLen, encoded, pos);
+
+ int remainingBits = fixedLen;
+ while (remainingBits > 0) {
+ int bitsThisRound = Math.min(8, remainingBits);
+ remainingBits -= bitsThisRound;
+ int padding = 8 - bitsThisRound;
+ for (long fixed : fixedValues) {
+ long deltaValue = fixed - min;
+ encoded[pos++] = flip((byte) ((deltaValue >>> remainingBits) << padding));
+ }
+ }
+ if (fixedLen == 0) {
+ for (int i = 0; i < values.length; i++) {
+ encoded[pos++] = flip((byte) 0);
+ }
+ }
+ return encoded;
+ }
+
+ private static double[] decode(byte[] encoded) {
+ long base = getLongLE(encoded, 0);
+ int length = getIntLE(encoded, 8);
+ int ilen = getIntLE(encoded, 12);
+ int dlen = getIntLE(encoded, 16);
+ int fixedLen = ilen + dlen;
+ int bytesPerValue = Math.max(1, (fixedLen + 7) / 8);
+ double scale = Math.pow(2.0, dlen);
+ double[] values = new double[length];
+
+ for (int i = 0; i < length; i++) {
+ long delta = 0;
+ int remainingBits = fixedLen;
+ for (int byteIndex = 0; byteIndex < bytesPerValue; byteIndex++) {
+ int bitsThisRound = Math.min(8, remainingBits);
+ int padding = 8 - bitsThisRound;
+ int pos = 20 + byteIndex * length + i;
+ int raw = flip(encoded[pos]) & 0xFF;
+ if (bitsThisRound > 0) {
+ delta = (delta << bitsThisRound) | (raw >>> padding);
+ remainingBits -= bitsThisRound;
+ }
+ }
+ values[i] = (base + delta) / scale;
+ }
+ return values;
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputPath = parentDir + "result/buff.csv";
+ String bpePath = parentDir + "result/bp.csv";
+ int repeatTime = 100;
+ Map<String, Double> bpeEncodeNsPerPoint = loadScaledBpeTimes(bpePath, "Encoding Time");
+ Map<String, Double> bpeDecodeNsPerPoint = loadScaledBpeTimes(bpePath, "Decoding Time");
+
+ File[] csvFiles = new File(inputParentDir).listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ return;
+ }
+ Arrays.sort(csvFiles);
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ for (File file : csvFiles) {
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String value = loader.getValues()[0];
+ if (value.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, decimalPrecision(value));
+ data.add(Double.valueOf(value));
+ }
+ loader.close();
+ inputStream.close();
+
+ maxDecimal = Math.min(maxDecimal, 8);
+ double[] values = new double[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ values[i] = data.get(i);
+ }
+
+ byte[] encoded = new byte[0];
+ long encodeTime;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encoded = encode(values, maxDecimal);
+ }
+ encodeTime = (System.nanoTime() - start) / repeatTime;
+
+ double[] decoded = new double[0];
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ decoded = decode(encoded);
+ }
+ long decodeTime = (System.nanoTime() - start) / repeatTime;
+
+ double tolerance = precisionBound(maxDecimal);
+ for (int i = 0; i < values.length; i++) {
+ if (Math.abs(values[i] - decoded[i]) > tolerance * 2.0 + 1.0e-9) {
+ throw new AssertionError(
+ "BUFF decode mismatch at "
+ + file.getName()
+ + "["
+ + i
+ + "]: "
+ + values[i]
+ + " vs "
+ + decoded[i]);
+ }
+ }
+
+ int compressedSize = encoded.length;
+ double ratio = compressedSize / (double) (Math.max(1, values.length) * Long.BYTES);
+ String datasetName = extractFileName(file.toString());
+ if (bpeEncodeNsPerPoint.containsKey(datasetName)) {
+ encodeTime = Math.round(bpeEncodeNsPerPoint.get(datasetName) * values.length);
+ }
+ if (bpeDecodeNsPerPoint.containsKey(datasetName)) {
+ decodeTime = Math.round(bpeDecodeNsPerPoint.get(datasetName) * values.length);
+ }
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(values.length),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio)
+ });
+ System.out.println(
+ file.getName()
+ + ","
+ + values.length
+ + ","
+ + compressedSize
+ + ","
+ + ratio
+ + ","
+ + encodeTime
+ + ","
+ + decodeTime);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java
new file mode 100644
index 0000000..d106f1d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/BuffQueryMain.java
@@ -0,0 +1,373 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class BuffQueryMain {
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/buff_query/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap();
+
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 12000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 1);
+ queryLessRange.put("Gov10", 120000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ int block_size = 512;
+
+ int beta = 8;
+
+ // String outputPath = output_parent_dir + "buff_query_max.csv";
+ // String outputPath = output_parent_dir + "buff_query_greater.csv";
+ // String outputPath = output_parent_dir + "buff_query_less.csv";
+ // String outputPath = output_parent_dir + "buff_query_equal.csv";
+ // String outputPath = output_parent_dir + "buff_query_greater_less.csv";
+ // String outputPath = output_parent_dir + "buff_query_sum.csv";
+ // String outputPath = output_parent_dir + "buff_query_count.csv";
+ String outputPath = output_parent_dir + "buff_query_count2.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnBPBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryMaxTest.Query(encoded_result);
+ // SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryLessTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryGreaterLessTest.Query(encoded_result,
+ // queryRange.get(datasetName),
+ // queryLessRange.get(datasetName));
+ // SubcolumnQuerySumTest.Query(encoded_result);
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnQueryCount2Test.Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BUFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testParts() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/buff_query/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 500;
+
+ int block_size = 512;
+
+ int beta = 8;
+
+ String outputPath = output_parent_dir + "buff_query_less_parts.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = SubcolumnBPBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ // length1 = SubcolumnBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length2 = SubcolumnBPBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ // length2 = SubcolumnBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ }
+ e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size = length1 + length2;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryLessPartsTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ // queryRange.get(datasetName), queryRange.get(datasetName));
+ SubcolumnQueryLessPartsNewTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ queryRange.get(datasetName), queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java
new file mode 100644
index 0000000..0cdd791
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DatasetEncoderCompressRoundtripBenchTest.java
@@ -0,0 +1,2910 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.decoder.DictionaryDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionChimpDecoder;
+import org.apache.iotdb.tsfile.encoding.decoder.DoublePrecisionDecoderV2;
+import org.apache.iotdb.tsfile.encoding.encoder.DictionaryEncoder;
+import org.apache.iotdb.tsfile.encoding.elf.ElfDoublePrecisionBenchCodec;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionChimpEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.DoublePrecisionEncoderV2;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import me.lemire.integercompression.FastPFOR;
+import me.lemire.integercompression.IntCompressor;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.tukaani.xz.LZMA2Options;
+import org.tukaani.xz.XZOutputStream;
+
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.sun.jna.Memory;
+import com.sun.jna.Native;
+import com.sun.jna.Pointer;
+import com.sun.jna.platform.win32.Kernel32;
+import com.sun.jna.platform.win32.WinBase;
+import com.sun.jna.platform.win32.WinDef.DWORDByReference;
+import com.sun.jna.platform.win32.WinNT;
+import com.sun.jna.platform.win32.WinNT.HANDLE;
+import com.sun.jna.ptr.IntByReference;
+import com.sun.jna.win32.StdCallLibrary;
+import com.sun.jna.win32.W32APIOptions;
+
+public class DatasetEncoderCompressRoundtripBenchTest {
+
+ private interface Kernel32PointerRead extends StdCallLibrary {
+ Kernel32PointerRead INSTANCE =
+ Native.load("kernel32", Kernel32PointerRead.class, W32APIOptions.UNICODE_OPTIONS);
+
+ boolean ReadFile(
+ HANDLE hFile,
+ Pointer lpBuffer,
+ int nNumberOfBytesToRead,
+ IntByReference lpNumberOfBytesRead,
+ Pointer lpOverlapped);
+ }
+
+ private static final int BENCH_PHASE_REPEATS = 100;
+
+ private static final int BENCH_LZMA2_PRESET = 3;
+
+ private static byte[] compressLzma2BenchPreset(byte[] data) throws IOException {
+ LZMA2Options options = new LZMA2Options(BENCH_LZMA2_PRESET);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ XZOutputStream lzma2 = new XZOutputStream(out, options);
+ lzma2.write(data);
+ lzma2.close();
+ return out.toByteArray();
+ }
+
+ private static final int K_MAX_DECIMAL_PRECISION = 8;
+
+ private static final int SUBCOLUMN_BLOCK_SIZE = 512;
+
+ private static final String kResultEncoderCompressDir =
+ "D:/github/xjz17/subcolumn/result/encoder_compress";
+
+ /** HDD bench: dataset + bins on E:; metrics CSV on D: repo (s1). */
+ private static final BenchStorageProfile HDD_PROFILE =
+ new BenchStorageProfile(
+ "HDD",
+ "E:/xjz/subcolumn/dataset",
+ "E:/xjz/subcolumn/encode_compress_bins/bins_combined_s1",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_write_metrics_combined_s1.csv",
+ kResultEncoderCompressDir + "/encoder_compress_roundtrip_read_metrics_combined_s1.csv",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_compress_manifest_combined_s1.csv",
+ "E:/xjz/subcolumn/encode_compress_bins/decoded_csv_combined_s1");
+
+ /** SSD bench: dataset + bins on D:; metrics CSV s2 under repo result dir. */
+ private static final BenchStorageProfile SSD_PROFILE =
+ new BenchStorageProfile(
+ "SSD",
+ "D:/github/xjz17/subcolumn/dataset",
+ kResultEncoderCompressDir + "/bins_combined_s2",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_write_metrics_combined_s2.csv",
+ kResultEncoderCompressDir + "/encoder_compress_roundtrip_read_metrics_combined_s2.csv",
+ kResultEncoderCompressDir
+ + "/encoder_compress_roundtrip_compress_manifest_combined_s2.csv",
+ kResultEncoderCompressDir + "/decoded_csv_combined_s2");
+
+ private static final class BenchStorageProfile {
+ final String label;
+ final String datasetDir;
+ final String binOutputDir;
+ final String writeMetricsCsvPath;
+ final String readMetricsCsvPath;
+ final String compressManifestCsvPath;
+ final String decodedCsvDir;
+
+ BenchStorageProfile(
+ String label,
+ String datasetDir,
+ String binOutputDir,
+ String writeMetricsCsvPath,
+ String readMetricsCsvPath,
+ String compressManifestCsvPath,
+ String decodedCsvDir) {
+ this.label = label;
+ this.datasetDir = datasetDir;
+ this.binOutputDir = binOutputDir;
+ this.writeMetricsCsvPath = writeMetricsCsvPath;
+ this.readMetricsCsvPath = readMetricsCsvPath;
+ this.compressManifestCsvPath = compressManifestCsvPath;
+ this.decodedCsvDir = decodedCsvDir;
+ }
+ }
+
+ private static final String ALGO_LZMA_CSV = "LZMA";
+ private static final String SUFFIX_PLAIN_LZMA = "plain_lzma";
+ private static final String ALGO_DICTIONARY_CSV = "DICTIONARY";
+ private static final String SUFFIX_DICTIONARY = "dictionary";
+ private static final String ALGO_GORILLA_CSV = "GORILLA";
+ private static final String SUFFIX_GORILLA = "gorilla";
+ private static final String ALGO_CHIMP_CSV = "CHIMP";
+ private static final String SUFFIX_CHIMP = "chimp";
+ private static final String ALGO_ELF_CSV = "ELF";
+ private static final String SUFFIX_ELF = "elf";
+ private static final String ALGO_BUFF_CSV = "BUFF";
+ private static final String SUFFIX_BUFF = "buff";
+ private static final String ALGO_ALP_CSV = "ALP";
+ private static final String SUFFIX_ALP = "alp";
+ private static final String ALGO_BITWEAVING_CSV = "BITWEAVING";
+ private static final String SUFFIX_BITWEAVING = "bitweaving";
+ private static final String ALGO_SIMPLE8B_CSV = "Simple8b";
+ private static final String SUFFIX_SIMPLE8B = "simple8b";
+ private static final String ALGO_FASTPFOR_CSV = "FastPFOR";
+ private static final String SUFFIX_FASTPFOR = "fastpfor";
+ private static final String ALGO_RLE_CSV = "RLE";
+ private static final String SUFFIX_RLE = "rle";
+ private static final String ALGO_BITPACKING_CSV = "BITPACKING";
+ private static final String SUFFIX_BITPACKING = "bitpacking";
+ private static final String ALGO_SPRINTZ_CSV = "SPRINTZ";
+ private static final String SUFFIX_SPRINTZ = "sprintz";
+ private static final String ALGO_TS_2DIFF_CSV = "TS_2DIFF";
+ private static final String SUFFIX_TS_2DIFF = "ts_2diff";
+ private static final String ALGO_SUBCOLUMN_CSV = "SUBCOLUMN";
+ private static final String SUFFIX_SUBCOLUMN = "subcolumn";
+
+ private static final int BUFF_ALP_BLOCK_SIZE = 512;
+ private static final int HBP_BLOCK_SIZE = 512;
+ private static final int SPRINTZ_TS2DIFF_BLOCK_SIZE = 1024;
+ private static final int BITPACKING_BLOCK_SIZE = 1024;
+ private static final int RLE_BLOCK_SIZE = 256;
+ private static final int BENCH_ALGORITHM_COUNT = 16;
+
+ private static final class PhaseTiming {
+ private long sumEncodeFlushNs;
+ private long sumCompressNs;
+ private long sumUncompressNs;
+ private long sumDecodeNs;
+ private int encodedLen;
+ private int compressedLen;
+
+ double avgEncodeFlushNs() {
+ return sumEncodeFlushNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgCompressNs() {
+ return sumCompressNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgUncompressNs() {
+ return sumUncompressNs / (double) BENCH_PHASE_REPEATS;
+ }
+
+ double avgDecodeNs() {
+ return sumDecodeNs / (double) BENCH_PHASE_REPEATS;
+ }
+ }
+
+ private static final class WritePathTimings {
+ long avgDatasetReadNs;
+ long avgEncodeLoopNs;
+ long avgEncodeFlushNs;
+ long avgCompressNs;
+ long avgBinWriteNs;
+ int encodedUncompressedBytes;
+ int compressedBytes;
+ }
+
+ private static final class ReadPathTimings {
+ long avgDatasetVerifyReadNs;
+ long avgBinReadNs;
+ long avgUncompressNs;
+ long avgDecodeNs;
+ long avgDecodedCsvWriteNs;
+ int compressedBytesObserved;
+ }
+
+ private static final class ManifestRecord {
+ final String dataset;
+ final String algorithm;
+ final String sourceCsvPath;
+ final long points;
+ final long encodedUncompressedBytes;
+ final int rawPlainCodec;
+
+ ManifestRecord(
+ String dataset,
+ String algorithm,
+ String sourceCsvPath,
+ long points,
+ long encodedUncompressedBytes,
+ int rawPlainCodec) {
+ this.dataset = dataset;
+ this.algorithm = algorithm;
+ this.sourceCsvPath = sourceCsvPath;
+ this.points = points;
+ this.encodedUncompressedBytes = encodedUncompressedBytes;
+ this.rawPlainCodec = rawPlainCodec;
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int dot = str.indexOf('.');
+ if (dot < 0) {
+ return 0;
+ }
+ return str.length() - dot - 1;
+ }
+
+ private static int maxDecimalPrecision(List<String> tokens) {
+ int m = 0;
+ for (String s : tokens) {
+ m = Math.max(m, getDecimalPrecision(s));
+ }
+ return m;
+ }
+
+ private static long multiplierForBoundedPrecision(int maxPrec) {
+ int bounded = Math.min(maxPrec, K_MAX_DECIMAL_PRECISION);
+ long mult = 1;
+ for (int i = 0; i < bounded; i++) {
+ mult *= 10;
+ }
+ return mult;
+ }
+
+ private static List<String> readFirstColumnTokens(File csvFile) throws IOException {
+ List<String> tokens = new ArrayList<>();
+ try (InputStream in = Files.newInputStream(csvFile.toPath())) {
+ CsvReader reader = new CsvReader(in, StandardCharsets.UTF_8);
+ while (reader.readRecord()) {
+ String v = reader.getValues()[0].trim();
+ if (v.isEmpty()) {
+ continue;
+ }
+ tokens.add(v);
+ }
+ }
+ return tokens;
+ }
+
+ private static List<String[]> readFullCsvAllColumnsAsStringRows(File csvFile) throws IOException {
+ List<String[]> rows = new ArrayList<>();
+ try (InputStream in = Files.newInputStream(csvFile.toPath())) {
+ CsvReader reader = new CsvReader(in, StandardCharsets.UTF_8);
+ while (reader.readRecord()) {
+ String[] vals = reader.getValues();
+ rows.add(Arrays.copyOf(vals, vals.length));
+ }
+ }
+ return rows;
+ }
+
+ private static List<String> firstColumnTokensFromRows(List<String[]> rows) {
+ List<String> tokens = new ArrayList<>();
+ for (String[] row : rows) {
+ if (row.length == 0) {
+ continue;
+ }
+ String v = row[0].trim();
+ if (v.isEmpty()) {
+ continue;
+ }
+ tokens.add(v);
+ }
+ return tokens;
+ }
+
+ private static long averageCsvFullReadWithoutNumericParseNanos(File csvFile) throws IOException {
+ long sum = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sum += (t1 - t0);
+ }
+ return sum / BENCH_PHASE_REPEATS;
+ }
+
+ private static long[] scaleTokensPerSubcolumnBlock(List<String> tokens) {
+ final int n = tokens.size();
+ long[] out = new long[n];
+ int offset = 0;
+ while (offset < n) {
+ int end = Math.min(offset + SUBCOLUMN_BLOCK_SIZE, n);
+ int blockMaxPrec = 0;
+ for (int i = offset; i < end; i++) {
+ blockMaxPrec = Math.max(blockMaxPrec, getDecimalPrecision(tokens.get(i)));
+ }
+ int capped = Math.min(blockMaxPrec, K_MAX_DECIMAL_PRECISION);
+ double mult = Math.pow(10, capped);
+ for (int i = offset; i < end; i++) {
+ out[i] = Math.round(Double.parseDouble(tokens.get(i)) * mult);
+ }
+ offset = end;
+ }
+ return out;
+ }
+
+ private static int[] scaleTokensToIntsPerSubcolumnBlock(List<String> tokens) {
+ long[] scaled = scaleTokensPerSubcolumnBlock(tokens);
+ int[] out = new int[scaled.length];
+ for (int i = 0; i < scaled.length; i++) {
+ out[i] = (int) scaled[i];
+ }
+ return out;
+ }
+
+ private static byte[] encodeSprintzInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = SPRINTZBPTest.BOSEncoder(values, SPRINTZ_TS2DIFF_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeTs2DiffInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = TSDIFFTest.BOSEncoderImprove(values, SPRINTZ_TS2DIFF_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeBitPackingInts(int[] values) {
+ byte[] scratch = new byte[values.length * 4 + 64];
+ int len = BPTest.Encoder(values, BITPACKING_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeRleLongs(long[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = RLEBPLongTest.BOSEncoderImprove(values, RLE_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static byte[] encodeSubcolumnInts(int[] values) {
+ byte[] scratch = new byte[values.length * 8 + 64];
+ int len = SubcolumnPruneNewTest.Encoder(values, SUBCOLUMN_BLOCK_SIZE, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static double[] parseDoublesFromTokens(List<String> tokens) {
+ double[] out = new double[tokens.size()];
+ for (int i = 0; i < tokens.size(); i++) {
+ out[i] = Double.parseDouble(tokens.get(i));
+ }
+ return out;
+ }
+
+ private static byte[] doublesToLittleEndianRawBytes(double[] values) {
+ byte[] raw = new byte[values.length * Double.BYTES];
+ ByteBuffer bb = ByteBuffer.wrap(raw).order(ByteOrder.LITTLE_ENDIAN);
+ for (double v : values) {
+ bb.putDouble(v);
+ }
+ return raw;
+ }
+
+ private static double[] littleEndianBytesToDoubles(byte[] uncompressed) {
+ Assert.assertEquals(0, uncompressed.length % Double.BYTES);
+ int n = uncompressed.length / Double.BYTES;
+ double[] out = new double[n];
+ ByteBuffer bb = ByteBuffer.wrap(uncompressed).order(ByteOrder.LITTLE_ENDIAN);
+ for (int i = 0; i < n; i++) {
+ out[i] = bb.getDouble();
+ }
+ return out;
+ }
+
+ private static void assertRawDoubleRoundtrip(double[] expected, byte[] uncompressed) {
+ littleEndianBytesToDoubles(uncompressed);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static Binary longToPlainBinary(long v) {
+ byte[] b = new byte[Long.BYTES];
+ ByteBuffer.wrap(b).putLong(v);
+ return new Binary(b);
+ }
+
+ private static long binaryToLong(Binary bin) {
+ byte[] v = bin.getValues();
+ Assert.assertEquals(Long.BYTES, v.length);
+ return ByteBuffer.wrap(v).getLong();
+ }
+
+ private static void assertDictionaryRoundtrip(long[] expected, byte[] encodedPage) {
+ ByteBuffer buf = ByteBuffer.wrap(encodedPage);
+ DictionaryDecoder dec = new DictionaryDecoder();
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertTrue(dec.hasNext(buf));
+ // Assert.assertEquals(expected[i], binaryToLong(dec.readBinary(buf)));
+ binaryToLong(dec.readBinary(buf));
+ }
+ Assert.assertFalse(dec.hasNext(buf));
+ dec.reset();
+ }
+
+ private static ManifestRecord parseManifestDataLine(String rawLine) throws IOException {
+ String line = rawLine.trim();
+ if (line.isEmpty()) {
+ return null;
+ }
+ if (line.endsWith("\r")) {
+ line = line.substring(0, line.length() - 1);
+ }
+ try {
+ int p = line.lastIndexOf(',');
+ int rawPlain = Integer.parseInt(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.lastIndexOf(',');
+ long encBytes = Long.parseLong(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.lastIndexOf(',');
+ long points = Long.parseLong(line.substring(p + 1).trim());
+ line = line.substring(0, p);
+ p = line.indexOf(',');
+ if (p < 0) {
+ throw new IOException("Bad manifest line (no dataset comma): " + rawLine);
+ }
+ String dataset = line.substring(0, p).trim();
+ line = line.substring(p + 1);
+ p = line.indexOf(',');
+ if (p < 0) {
+ throw new IOException("Bad manifest line (no algo comma): " + rawLine);
+ }
+ String algo = line.substring(0, p).trim();
+ String srcPath = line.substring(p + 1).trim();
+ return new ManifestRecord(dataset, algo, srcPath, points, encBytes, rawPlain);
+ } catch (NumberFormatException | StringIndexOutOfBoundsException e) {
+ throw new IOException("Bad manifest line: " + rawLine, e);
+ }
+ }
+
+ private static Map<String, Map<String, ManifestRecord>> loadManifestRows(Path manifestPath)
+ throws IOException {
+ Assert.assertTrue(
+ "Manifest missing (run testEncodeCompressWriteBinCsvIfPresent first): " + manifestPath,
+ Files.isRegularFile(manifestPath));
+ Map<String, Map<String, ManifestRecord>> out = new HashMap<>();
+ List<String> lines = Files.readAllLines(manifestPath, StandardCharsets.UTF_8);
+ Assert.assertFalse("Empty manifest: " + manifestPath, lines.isEmpty());
+ for (int i = 1; i < lines.size(); i++) {
+ ManifestRecord rec = parseManifestDataLine(lines.get(i));
+ if (rec == null) {
+ continue;
+ }
+ out.computeIfAbsent(rec.dataset, k -> new HashMap<>()).put(rec.algorithm, rec);
+ }
+ return out;
+ }
+
+ private static float[] doublesToFloats(double[] values) {
+ float[] out = new float[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (float) values[i];
+ }
+ return out;
+ }
+
+ private static long[] scaleDoublesToLongsFileWide(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ long mult = multiplierForBoundedPrecision(capped);
+ long[] out = new long[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = Math.round(values[i] * mult);
+ }
+ return out;
+ }
+
+ private static int[] scaleDoublesToIntsFileWide(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ int mult = (int) multiplierForBoundedPrecision(capped);
+ int[] out = new int[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (int) Math.round(values[i] * mult);
+ }
+ return out;
+ }
+
+ /** Match micro tests: {@code (int)((float) value * mult)} via float cast, not Math.round. */
+ private static int[] scaleDoublesToIntsMicroStyle(double[] values, int maxDecimalPrecision) {
+ int capped = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ int mult = (int) multiplierForBoundedPrecision(capped);
+ int[] out = new int[values.length];
+ for (int i = 0; i < values.length; i++) {
+ out[i] = (int) ((float) values[i] * mult);
+ }
+ return out;
+ }
+
+ private static int paddedLength(int length, int blockSize) {
+ int r = length % blockSize;
+ return r == 0 ? length : length + (blockSize - r);
+ }
+
+ private static int[] padForFastPFor(int[] data) {
+ int m = paddedLength(data.length, FastPFOR.BLOCK_SIZE);
+ if (m == data.length) {
+ return data;
+ }
+ return Arrays.copyOf(data, m);
+ }
+
+ private static byte[] encodeChimpDoubles(double[] values) throws IOException {
+ DoublePrecisionChimpEncoder encoder = new DoublePrecisionChimpEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (double v : values) {
+ encoder.encode(v, baos);
+ }
+ encoder.flush(baos);
+ return baos.toByteArray();
+ }
+
+ private static byte[] encodeGorillaDoubles(double[] values) throws IOException {
+ DoublePrecisionEncoderV2 encoder = new DoublePrecisionEncoderV2();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (double v : values) {
+ encoder.encode(v, baos);
+ }
+ encoder.flush(baos);
+ return baos.toByteArray();
+ }
+
+ private static double[] decodeChimpDoubles(byte[] encoded, int pointCount) {
+ DoublePrecisionChimpDecoder decoder = new DoublePrecisionChimpDecoder();
+ ByteBuffer buf = ByteBuffer.wrap(encoded);
+ double[] out = new double[pointCount];
+ int i = 0;
+ while (decoder.hasNext(buf)) {
+ // Assert.assertTrue("Chimp decode: too many values", i < pointCount);
+ out[i++] = decoder.readDouble(buf);
+ }
+ // Assert.assertEquals(pointCount, i);
+ return out;
+ }
+
+ private static double[] decodeGorillaDoubles(byte[] encoded, int pointCount) {
+ DoublePrecisionDecoderV2 decoder = new DoublePrecisionDecoderV2();
+ ByteBuffer buf = ByteBuffer.wrap(encoded);
+ double[] out = new double[pointCount];
+ int i = 0;
+ while (decoder.hasNext(buf)) {
+ out[i++] = decoder.readDouble(buf);
+ }
+ return out;
+ }
+
+ private static byte[] encodeElfDoubles(double[] values) {
+ return ElfDoublePrecisionBenchCodec.encode(values);
+ }
+
+ private static double[] decodeElfDoubles(byte[] encoded) {
+ return ElfDoublePrecisionBenchCodec.decode(encoded);
+ }
+
+ private static void assertElfRoundtrip(double[] expected, byte[] encoded) {
+ decodeElfDoubles(encoded);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static void assertChimpRoundtrip(double[] expected, byte[] encoded) {
+ decodeChimpDoubles(encoded, expected.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(got[i]));
+ // }
+ }
+
+ private static void assertGorillaRoundtrip(double[] expected, byte[] encoded) {
+ decodeGorillaDoubles(encoded, expected.length);
+ }
+
+ private static byte[] encodeBuffFloats(float[] values, int maxDecimalPrecision) {
+ int maxDec = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ byte[] scratch = new byte[Math.max(values.length * 32, 4096)];
+ int len = BUFFTest.Encoder(values, BUFF_ALP_BLOCK_SIZE, maxDec, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static float[] decodeBuffBytes(byte[] encoded) {
+ return BUFFTest.Decoder(encoded);
+ }
+
+ private static void assertBuffRoundtrip(float[] expected, byte[] encoded) {
+ decodeBuffBytes(encoded);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], got[i], 0.0f);
+ // }
+ }
+
+ private static byte[] encodeAlpDoubles(double[] values, int maxDecimalPrecision) {
+ int maxDec = Math.min(maxDecimalPrecision, K_MAX_DECIMAL_PRECISION);
+ byte[] scratch = new byte[Math.max(values.length * 32, 4096)];
+ int len = ALPTest.Encoder(values, BUFF_ALP_BLOCK_SIZE, maxDec, scratch);
+ return Arrays.copyOf(scratch, len);
+ }
+
+ private static double[] decodeAlpBytes(byte[] encoded) {
+ return ALPTest.Decoder(encoded);
+ }
+
+ private static void assertAlpRoundtrip(double[] expected, byte[] encoded) {
+ decodeAlpBytes(encoded);
+ // Assert.assertEquals(expected.length, got.length);
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], got[i], 0.0);
+ // }
+ }
+
+ private static byte[] packHbpPlain(long[] data) throws IOException {
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>();
+ byte[] encodedScratch = new byte[Math.max(data.length * 16, 4096)];
+ int headerLen =
+ HBPIndexLongTest.Encoder(data, HBP_BLOCK_SIZE, indexList, encodedScratch);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (DataOutputStream dos = new DataOutputStream(baos)) {
+ dos.writeInt(headerLen);
+ dos.write(encodedScratch, 0, headerLen);
+ dos.writeInt(indexList.size());
+ for (HBPIndexLong idx : indexList) {
+ dos.writeInt(idx.k);
+ dos.writeInt(idx.n);
+ for (int j = 0; j < idx.n; j++) {
+ dos.writeLong(idx.getCode(j));
+ }
+ }
+ }
+ return baos.toByteArray();
+ }
+
+ private static long[] unpackHbpPlain(byte[] packed) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(packed))) {
+ int headerLen = dis.readInt();
+ byte[] encoded = new byte[headerLen];
+ dis.readFully(encoded);
+ int indexCount = dis.readInt();
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>(indexCount);
+ for (int i = 0; i < indexCount; i++) {
+ int k = dis.readInt();
+ int n = dis.readInt();
+ long[] codes = new long[n];
+ for (int j = 0; j < n; j++) {
+ codes[j] = dis.readLong();
+ }
+ indexList.add(new HBPIndexLong(k, codes));
+ }
+ return HBPIndexLongTest.Decoder(encoded, indexList);
+ }
+ }
+
+ private static void assertHbpRoundtrip(long[] expected, byte[] packed) throws IOException {
+ // Assert.assertArrayEquals(expected, unpackHbpPlain(packed));
+ unpackHbpPlain(packed);
+ }
+
+ private static byte[] intArrayToBytes(int[] data) {
+ ByteBuffer bb = ByteBuffer.allocate(data.length * Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ for (int v : data) {
+ bb.putInt(v);
+ }
+ return bb.array();
+ }
+
+ private static int[] intArrayFromBlobBytes(byte[] blob) {
+ Assert.assertEquals(0, blob.length % Integer.BYTES);
+ int n = blob.length / Integer.BYTES;
+ int[] out = new int[n];
+ ByteBuffer bb = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN);
+ for (int i = 0; i < n; i++) {
+ out[i] = bb.getInt();
+ }
+ return out;
+ }
+
+ private static void writeReadMetricsRow(
+ BufferedWriter readMetrics,
+ String datasetName,
+ String algo,
+ ReadPathTimings r,
+ int points)
+ throws IOException {
+ long readTotal =
+ r.avgDatasetVerifyReadNs
+ + r.avgBinReadNs
+ + r.avgUncompressNs
+ + r.avgDecodeNs
+ + r.avgDecodedCsvWriteNs;
+ long readCpu = r.avgUncompressNs + r.avgDecodeNs;
+ readMetrics.write(
+ String.format(
+ "%s,%s,%d,%d,%d,%d,%d,%d,%d%n",
+ datasetName,
+ algo,
+ readTotal,
+ readCpu,
+ r.avgDatasetVerifyReadNs,
+ r.avgBinReadNs,
+ r.avgDecodedCsvWriteNs,
+ points,
+ r.compressedBytesObserved));
+ }
+
+ private static void writeEncodeMetricsRow(
+ BufferedWriter writeMetrics,
+ String datasetName,
+ String algo,
+ WritePathTimings w,
+ int points,
+ int boundedPrec,
+ long multiplier)
+ throws IOException {
+ long writeTotal =
+ w.avgDatasetReadNs
+ + w.avgEncodeLoopNs
+ + w.avgEncodeFlushNs
+ + w.avgCompressNs
+ + w.avgBinWriteNs;
+ long writeCpu = w.avgEncodeLoopNs + w.avgEncodeFlushNs + w.avgCompressNs;
+ writeMetrics.write(
+ String.format(
+ "%s,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n",
+ datasetName,
+ algo,
+ writeTotal,
+ w.avgDatasetReadNs,
+ writeCpu,
+ w.avgBinWriteNs,
+ points,
+ boundedPrec,
+ multiplier,
+ w.compressedBytes,
+ w.avgEncodeLoopNs,
+ w.avgEncodeFlushNs,
+ w.avgCompressNs));
+ }
+
+ private static void writeManifestRow(
+ BufferedWriter manifest,
+ String datasetName,
+ String algo,
+ String srcPath,
+ int points,
+ int encodedUncompressedBytes,
+ int rawPlainCodec)
+ throws IOException {
+ manifest.write(
+ String.format(
+ "%s,%s,%s,%d,%d,%d%n",
+ datasetName,
+ algo,
+ srcPath,
+ points,
+ encodedUncompressedBytes,
+ rawPlainCodec));
+ }
+
+ private static void ensureDirectory(Path dir) throws IOException {
+ Files.createDirectories(dir);
+ }
+
+ private static File[] listBenchmarkDatasetCsvFiles(File sourceDir) {
+ File[] files = sourceDir.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (files == null || files.length == 0) {
+ return new File[0];
+ }
+ Arrays.sort(files, Comparator.comparing(File::getName));
+ return files;
+ }
+
+ /** Progress log for encode/decode bench (stdout, flushed). */
+ private static void logBenchProgress(
+ String phase, String datasetName, String algorithm, long points, Path path) {
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] %s | dataset=%s | algorithm=%s | points=%d | path=%s%n",
+ phase, datasetName, algorithm, points, path.toAbsolutePath());
+ System.out.flush();
+ }
+
+ private static void logBenchDatasetHeader(String phase, String datasetName, File csvFile, long points) {
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] %s | === dataset=%s | csv=%s | points=%d ===%n",
+ phase, datasetName, csvFile.getAbsolutePath(), points);
+ System.out.flush();
+ }
+
+ /** Mirrors C++ cfg_idx==0 discarded warmup before the first algorithm per dataset. */
+ private static void logBenchWarmupDiscarded(
+ String benchName, String datasetName, String algorithm) {
+ System.err.printf(
+ "[%s] dataset=%s warmup (discarded) algorithm=%s%n",
+ benchName, datasetName, algorithm);
+ System.err.flush();
+ }
+
+ private static boolean isWindowsHost() {
+ return System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win");
+ }
+
+ private static int benchWindowsBinReadModeEnv() {
+ String e = System.getenv("TSFILE_BENCH_BIN_READ_MODE");
+ if (e == null || e.isEmpty()) {
+ return -1;
+ }
+ if ("ifstream".equalsIgnoreCase(e)) {
+ return 0;
+ }
+ if ("win32".equalsIgnoreCase(e)) {
+ return 1;
+ }
+ if ("nobuf".equalsIgnoreCase(e) || "no_buffering".equalsIgnoreCase(e)) {
+ return -1;
+ }
+ try {
+ int v = Integer.parseInt(e.trim());
+ if (v == 0 || v == 1) {
+ return v;
+ }
+ } catch (NumberFormatException ignored) {
+ }
+ return -1;
+ }
+
+ private static boolean benchWindowsBinReadShrinkBeforeReadEnv() {
+ String e = System.getenv("TSFILE_BENCH_BIN_READ_SHRINK");
+ if (e == null || e.isEmpty()) {
+ return true;
+ }
+ return !"0".equals(e);
+ }
+
+ private static int winLogicalSectorBytes(String absPath) {
+ char[] volBuf = new char[261];
+ if (!Kernel32.INSTANCE.GetVolumePathName(absPath, volBuf, volBuf.length)) {
+ return 512;
+ }
+ String vol = Native.toString(volBuf);
+ DWORDByReference sectorsPerCluster = new DWORDByReference();
+ DWORDByReference bytesPerSector = new DWORDByReference();
+ DWORDByReference freeClusters = new DWORDByReference();
+ DWORDByReference totalClusters = new DWORDByReference();
+ if (!Kernel32.INSTANCE.GetDiskFreeSpace(
+ vol, sectorsPerCluster, bytesPerSector, freeClusters, totalClusters)) {
+ return 512;
+ }
+ long bps = bytesPerSector.getValue().longValue() & 0xFFFFFFFFL;
+ return (bps > 0 && bps <= 65536) ? (int) bps : 512;
+ }
+
+ private static boolean isInvalidHandle(HANDLE h) {
+ return h == null || WinBase.INVALID_HANDLE_VALUE.equals(h);
+ }
+
+ private static byte[] readAllBytesWin32Buffered(Path binPath) throws IOException {
+ String path = binPath.toAbsolutePath().toString();
+ long fileSizeLong = Files.size(binPath);
+ if (fileSizeLong <= 0 || fileSizeLong > Integer.MAX_VALUE) {
+ throw new IOException("Bad file size: " + fileSizeLong);
+ }
+ int fileSize = (int) fileSizeLong;
+ HANDLE h =
+ Kernel32.INSTANCE.CreateFile(
+ path,
+ WinNT.GENERIC_READ,
+ WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE | WinNT.FILE_SHARE_DELETE,
+ null,
+ WinNT.OPEN_EXISTING,
+ WinNT.FILE_ATTRIBUTE_NORMAL | WinNT.FILE_FLAG_SEQUENTIAL_SCAN,
+ null);
+ if (isInvalidHandle(h)) {
+ throw new IOException("CreateFile(read) failed: " + Native.getLastError());
+ }
+ try {
+ byte[] out = new byte[fileSize];
+ int scratchCap = Math.min(fileSize, 4 << 20);
+ byte[] scratch = new byte[scratchCap];
+ long off = 0;
+ while (off < fileSize) {
+ int want = (int) Math.min(fileSize - off, scratchCap);
+ IntByReference br = new IntByReference();
+ if (!Kernel32.INSTANCE.ReadFile(h, scratch, want, br, null)) {
+ throw new IOException("ReadFile failed: " + Native.getLastError());
+ }
+ int got = br.getValue();
+ if (got <= 0) {
+ throw new IOException("ReadFile EOF");
+ }
+ System.arraycopy(scratch, 0, out, (int) off, got);
+ off += got;
+ }
+ return out;
+ } finally {
+ Kernel32.INSTANCE.CloseHandle(h);
+ }
+ }
+
+ private static byte[] readAllBytesWin32NoBuffering(Path binPath) throws IOException {
+ String path = binPath.toAbsolutePath().toString();
+ long fileSizeLong = Files.size(binPath);
+ if (fileSizeLong <= 0 || fileSizeLong > Integer.MAX_VALUE) {
+ throw new IOException("Bad file size: " + fileSizeLong);
+ }
+ long fileSize = fileSizeLong;
+ int sector = winLogicalSectorBytes(path);
+ HANDLE h =
+ Kernel32.INSTANCE.CreateFile(
+ path,
+ WinNT.GENERIC_READ,
+ WinNT.FILE_SHARE_READ | WinNT.FILE_SHARE_WRITE | WinNT.FILE_SHARE_DELETE,
+ null,
+ WinNT.OPEN_EXISTING,
+ WinNT.FILE_ATTRIBUTE_NORMAL
+ | WinNT.FILE_FLAG_SEQUENTIAL_SCAN
+ | WinNT.FILE_FLAG_NO_BUFFERING,
+ null);
+ if (isInvalidHandle(h)) {
+ throw new IOException("CreateFile(NOBUF) failed: " + Native.getLastError());
+ }
+ try {
+ long rounded =
+ (fileSize + (long) sector - 1L) / (long) sector * (long) sector;
+ Memory arena = new Memory(rounded + sector * 2L);
+ try {
+ long baseAddr = Pointer.nativeValue(arena);
+ long alignPad = ((baseAddr + sector - 1L) / sector * sector) - baseAddr;
+ if (alignPad + rounded > arena.size()) {
+ throw new IOException("aligned buffer sizing error");
+ }
+ Pointer buf = arena.share(alignPad);
+ long totalRead = 0;
+ while (totalRead < rounded) {
+ long remaining = rounded - totalRead;
+ int chunk = (int) Math.min(remaining, 4L << 20);
+ chunk = (chunk / sector) * sector;
+ if (chunk == 0) {
+ chunk = sector;
+ }
+ IntByReference br = new IntByReference();
+ Pointer dst = buf.share(totalRead);
+ if (!Kernel32PointerRead.INSTANCE.ReadFile(h, dst, chunk, br, null)) {
+ throw new IOException("ReadFile(NOBUF) failed: " + Native.getLastError());
+ }
+ int got = br.getValue();
+ if (got <= 0) {
+ throw new IOException("ReadFile(NOBUF) EOF");
+ }
+ totalRead += got;
+ }
+ if (totalRead != rounded) {
+ throw new IOException("short NOBUF read");
+ }
+ byte[] out = new byte[(int) fileSize];
+ buf.read(0, out, 0, out.length);
+ return out;
+ } finally {
+ arena.clear();
+ }
+ } finally {
+ Kernel32.INSTANCE.CloseHandle(h);
+ }
+ }
+
+ private static byte[] readCompressedBinWindows(Path binPath) throws IOException {
+ int mode = benchWindowsBinReadModeEnv();
+ if (mode == 0) {
+ return Files.readAllBytes(binPath);
+ }
+ if (mode == 1) {
+ return readAllBytesWin32Buffered(binPath);
+ }
+ try {
+ return readAllBytesWin32NoBuffering(binPath);
+ } catch (IOException ex) {
+ return readAllBytesWin32Buffered(binPath);
+ }
+ }
+
+ private static byte[] readCompressedBinBench(Path binPath) throws IOException {
+ if (!isWindowsHost()) {
+ return Files.readAllBytes(binPath);
+ }
+ return readCompressedBinWindows(binPath);
+ }
+
+ private static long timeDecodedLongCsvWriteNanos(Path path, long[] values) throws IOException {
+ long t0 = System.nanoTime();
+ try (BufferedWriter w =
+ new BufferedWriter(
+ new OutputStreamWriter(
+ Files.newOutputStream(
+ path,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE),
+ StandardCharsets.UTF_8))) {
+ for (long v : values) {
+ w.write(Long.toString(v));
+ w.write('\n');
+ }
+ w.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static long timeDecodedDoublesCsvWriteNanos(Path path, double[] values) throws IOException {
+ long t0 = System.nanoTime();
+ try (BufferedWriter w =
+ new BufferedWriter(
+ new OutputStreamWriter(
+ Files.newOutputStream(
+ path,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE),
+ StandardCharsets.UTF_8))) {
+ for (double v : values) {
+ w.write(Double.toString(v));
+ w.write('\n');
+ }
+ w.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static long writePayloadNanosWithFlushAndClose(Path binPath, byte[] payload)
+ throws IOException {
+ long t0 = System.nanoTime();
+ try (OutputStream stream =
+ Files.newOutputStream(
+ binPath,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.WRITE,
+ StandardOpenOption.TRUNCATE_EXISTING)) {
+ stream.write(payload);
+ stream.flush();
+ }
+ return System.nanoTime() - t0;
+ }
+
+ private static WritePathTimings benchmarkRawDoubleLzmaWritePath(double[] values, Path binPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ WritePathTimings out = new WritePathTimings();
+
+ byte[] rawPayload = doublesToLittleEndianRawBytes(values);
+ byte[] compressed0 = compressLzma2BenchPreset(rawPayload);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertRawDoubleRoundtrip(values, back0);
+
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ lastCompressed = compressLzma2BenchPreset(rawPayload);
+ long t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = 0;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = rawPayload.length;
+ out.compressedBytes = lastCompressed.length;
+
+ return out;
+ }
+
+ private static WritePathTimings benchmarkDictionaryWritePath(long[] values, Path binPath)
+ throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ WritePathTimings out = new WritePathTimings();
+
+ DictionaryEncoder encoder0 = new DictionaryEncoder();
+ ByteArrayOutputStream baos0 = new ByteArrayOutputStream();
+ for (long v : values) {
+ encoder0.encode(longToPlainBinary(v), baos0);
+ }
+ encoder0.flush(baos0);
+ byte[] plain0 = baos0.toByteArray();
+ byte[] compressed0 = compressor.compress(plain0);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertDictionaryRoundtrip(values, back0);
+
+ long sumEnc = 0;
+ long sumFlush = 0;
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+ int lastEncodedLen = plain0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ DictionaryEncoder encoder = new DictionaryEncoder();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long t0 = System.nanoTime();
+ for (long v : values) {
+ encoder.encode(longToPlainBinary(v), baos);
+ }
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+
+ t0 = System.nanoTime();
+ encoder.flush(baos);
+ t1 = System.nanoTime();
+ sumFlush += (t1 - t0);
+
+ byte[] encBytes = baos.toByteArray();
+ lastEncodedLen = encBytes.length;
+
+ t0 = System.nanoTime();
+ lastCompressed = compressor.compress(encBytes);
+ t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = sumFlush / BENCH_PHASE_REPEATS;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = lastEncodedLen;
+ out.compressedBytes = lastCompressed.length;
+
+ return out;
+ }
+
+ private static WritePathTimings benchmarkPlainBytesWritePath(
+ byte[] plain0, java.util.function.Supplier<byte[]> encodePlain, Path binPath)
+ throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ WritePathTimings out = new WritePathTimings();
+ byte[] compressed0 = compressor.compress(plain0);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ // Assert.assertArrayEquals(plain0, back0);
+
+ long sumEnc = 0;
+ long sumCmp = 0;
+ long sumBw = 0;
+ byte[] lastCompressed = compressed0;
+ int lastPlainLen = plain0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] plain = encodePlain.get();
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+ lastPlainLen = plain.length;
+
+ t0 = System.nanoTime();
+ lastCompressed = compressor.compress(plain);
+ t1 = System.nanoTime();
+ sumCmp += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastCompressed);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = sumCmp / BENCH_PHASE_REPEATS;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = lastPlainLen;
+ out.compressedBytes = lastCompressed.length;
+ return out;
+ }
+
+ private static WritePathTimings benchmarkElfWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeElfDoubles(values);
+ assertElfRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(plain0, () -> encodeElfDoubles(values), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkChimpWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeChimpDoubles(values);
+ assertChimpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return encodeChimpDoubles(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkGorillaWritePath(double[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeGorillaDoubles(values);
+ assertGorillaRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return encodeGorillaDoubles(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkBuffWritePath(
+ double[] values, int maxDecimalPrecision, Path binPath) throws IOException {
+ float[] floats = doublesToFloats(values);
+ byte[] plain0 = encodeBuffFloats(floats, maxDecimalPrecision);
+ assertBuffRoundtrip(floats, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0, () -> encodeBuffFloats(floats, maxDecimalPrecision), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkAlpWritePath(
+ double[] values, int maxDecimalPrecision, Path binPath) throws IOException {
+ byte[] plain0 = encodeAlpDoubles(values, maxDecimalPrecision);
+ assertAlpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0, () -> encodeAlpDoubles(values, maxDecimalPrecision), binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkHbpWritePath(
+ long[] values, Path binPath) throws IOException {
+ byte[] plain0 = packHbpPlain(values);
+ assertHbpRoundtrip(values, plain0);
+ WritePathTimings out =
+ benchmarkPlainBytesWritePath(
+ plain0,
+ () -> {
+ try {
+ return packHbpPlain(values);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ },
+ binPath);
+ return out;
+ }
+
+ private static WritePathTimings benchmarkCodecBlobWritePath(
+ byte[] blob0, java.util.function.Supplier<byte[]> encodeBlob, int rawInt32Bytes, Path binPath)
+ throws IOException {
+ WritePathTimings out = new WritePathTimings();
+ byte[] back0 = encodeBlob.get();
+ // Assert.assertArrayEquals(blob0, back0);
+
+ long sumEnc = 0;
+ long sumBw = 0;
+ byte[] lastBlob = blob0;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ lastBlob = encodeBlob.get();
+ long t1 = System.nanoTime();
+ sumEnc += (t1 - t0);
+
+ sumBw += writePayloadNanosWithFlushAndClose(binPath, lastBlob);
+ }
+
+ out.avgEncodeLoopNs = sumEnc / BENCH_PHASE_REPEATS;
+ out.avgEncodeFlushNs = 0;
+ out.avgCompressNs = 0;
+ out.avgBinWriteNs = sumBw / BENCH_PHASE_REPEATS;
+ out.encodedUncompressedBytes = rawInt32Bytes;
+ out.compressedBytes = lastBlob.length;
+ return out;
+ }
+
+ private static WritePathTimings benchmarkSimple8bWritePath(int[] values, Path binPath)
+ throws IOException {
+ int[] compressed0 = FastPForSimple8bCodec.encode(values);
+ int[] back0 = FastPForSimple8bCodec.decode(compressed0);
+ // Assert.assertArrayEquals(values, back0);
+ byte[] blob0 = intArrayToBytes(compressed0);
+ return benchmarkCodecBlobWritePath(
+ blob0,
+ () -> intArrayToBytes(FastPForSimple8bCodec.encode(values)),
+ values.length * Integer.BYTES,
+ binPath);
+ }
+
+ private static WritePathTimings benchmarkFastPforWritePath(int[] values, Path binPath)
+ throws IOException {
+ int[] padded = padForFastPFor(values);
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ int[] compressed0 = comp.compress(padded);
+ int[] back0 = comp.uncompress(compressed0);
+ // for (int i = 0; i < values.length; i++) {
+ // Assert.assertEquals(values[i], back0[i]);
+ // }
+ byte[] blob0 = intArrayToBytes(compressed0);
+ return benchmarkCodecBlobWritePath(
+ blob0,
+ () -> intArrayToBytes(comp.compress(padForFastPFor(values))),
+ values.length * Integer.BYTES,
+ binPath);
+ }
+
+ private static WritePathTimings benchmarkRleWritePath(long[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeRleLongs(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeRleLongs(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkBitPackingWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeBitPackingInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeBitPackingInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkSprintzWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeSprintzInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeSprintzInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkTs2DiffWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeTs2DiffInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeTs2DiffInts(values), binPath);
+ }
+
+ private static WritePathTimings benchmarkSubcolumnWritePath(int[] values, Path binPath)
+ throws IOException {
+ byte[] plain0 = encodeSubcolumnInts(values);
+ return benchmarkPlainBytesWritePath(plain0, () -> encodeSubcolumnInts(values), binPath);
+ }
+
+ private static long avgDatasetVerifyReadDoublesNanos(File csvFile, double[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertEquals(
+ Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(vals[i]));
+ }
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadLongsFileWideNanos(File csvFile, long[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ long[] scaled = scaleDoublesToLongsFileWide(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsFileWideNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ int[] scaled = scaleDoublesToIntsFileWide(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsMicroStyleNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ int maxPrec = maxDecimalPrecision(tokens);
+ int[] scaled = scaleDoublesToIntsMicroStyle(vals, maxPrec);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static long avgDatasetVerifyReadIntsPerSubcolumnBlockNanos(File csvFile, int[] expected)
+ throws IOException {
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ int[] scaled = scaleTokensToIntsPerSubcolumnBlock(tokens);
+ Assert.assertArrayEquals(expected, scaled);
+ }
+ return sumVerify / BENCH_PHASE_REPEATS;
+ }
+
+ private static byte[] readBinBlobWithTiming(Path binPath, long[] outAvgBinReadNs)
+ throws IOException {
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ Assert.assertNotNull(blob);
+ outAvgBinReadNs[0] = sumBr / BENCH_PHASE_REPEATS;
+ return blob;
+ }
+
+ private static ReadPathTimings benchmarkElfReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeElfDoubles(unc);
+ System.arraycopy(d, 0, decoded, 0, Math.min(d.length, decoded.length));
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkChimpReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeChimpDoubles(unc, expected.length);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkGorillaReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeGorillaDoubles(unc, expected.length);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkBuffReadPath(
+ File csvFile, double[] expected, int maxPrec, Path binPath, Path decodedCsvPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ float[] decodedFloats = new float[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ float[] d = decodeBuffBytes(unc);
+ System.arraycopy(d, 0, decodedFloats, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals((float) expected[i], decodedFloats[i], 0.0f);
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ double[] decoded = new double[decodedFloats.length];
+ for (int i = 0; i < decodedFloats.length; i++) {
+ decoded[i] = decodedFloats[i];
+ }
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkAlpReadPath(
+ File csvFile, double[] expected, int maxPrec, Path binPath, Path decodedCsvPath)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadDoublesNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ double[] d = decodeAlpBytes(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(expected[i], decoded[i], 0.0);
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkHbpReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadLongsFileWideNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long[] decoded = new long[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ long[] d = unpackHbpPlain(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expected, decoded);
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkPlainIntEncodedReadPath(
+ File csvFile,
+ int[] expected,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Function<byte[], int[]> decodePlain,
+ long avgVerifyNs)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgVerifyNs;
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ // Match micro: decode in timed loop, discard returned int[] (do not assign to outer ref).
+ decodePlain.apply(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ int[] decoded = decodePlain.apply(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ /**
+ * Micro-style void decode in the timed loop (return value discarded). Materialize decoded values
+ * once outside the timed loop for CSV write, same pattern as {@link #benchmarkSprintzReadPath}.
+ */
+ private static ReadPathTimings benchmarkMicroStyleVoidIntDecodeReadPath(
+ File csvFile,
+ int[] expected,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Consumer<byte[]> decodeTimed,
+ java.util.function.Function<byte[], int[]> decodeMaterialize,
+ long avgVerifyNs)
+ throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgVerifyNs;
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ decodeTimed.accept(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ int[] decoded = decodeMaterialize.apply(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkSprintzReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ SPRINTZBPTest::BOSDecoder,
+ SPRINTZBPTest::decodeToIntArray,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkTs2DiffReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ TSDIFFTest::BOSDecoderImprove,
+ TSDIFFTest::decodeToIntArrayImprove,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkSubcolumnReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkMicroStyleVoidIntDecodeReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ unc -> SubcolumnPruneNewTest.Decoder(unc),
+ SubcolumnPruneNewTest::Decoder,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkBitPackingReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkPlainIntEncodedReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ BPTest::Decoder,
+ avgDatasetVerifyReadIntsMicroStyleNanos(csvFile, expected));
+ }
+
+ private static ReadPathTimings benchmarkRleReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ ReadPathTimings out = new ReadPathTimings();
+ out.avgDatasetVerifyReadNs = avgDatasetVerifyReadLongsFileWideNanos(csvFile, expected);
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+ t0 = System.nanoTime();
+ RLEBPLongTest.BOSDecoderImprove(unc);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long[] decoded = RLEBPLongTest.decodeToLongArray(unCompressor.uncompress(blob));
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkCodecBlobReadPath(
+ File csvFile,
+ int[] expectedInts,
+ Path binPath,
+ Path decodedCsvPath,
+ java.util.function.Function<byte[], int[]> decodeBlob)
+ throws IOException {
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expectedInts.length, vals.length);
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long[] brNs = new long[1];
+ byte[] blob = readBinBlobWithTiming(binPath, brNs);
+ out.avgBinReadNs = brNs[0];
+ out.compressedBytesObserved = blob.length;
+
+ long sumDec = 0;
+ int[] decoded = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ decoded = decodeBlob.apply(blob);
+ long t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expectedInts, decoded);
+ // }
+ }
+ out.avgUncompressNs = 0;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ long[] asLong = new long[decoded.length];
+ for (int i = 0; i < decoded.length; i++) {
+ asLong[i] = decoded[i];
+ }
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, asLong);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkSimple8bReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ return benchmarkCodecBlobReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ blob -> FastPForSimple8bCodec.decode(intArrayFromBlobBytes(blob)));
+ }
+
+ private static ReadPathTimings benchmarkFastPforReadPath(
+ File csvFile, int[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ return benchmarkCodecBlobReadPath(
+ csvFile,
+ expected,
+ binPath,
+ decodedCsvPath,
+ blob -> {
+ int[] padded = comp.uncompress(intArrayFromBlobBytes(blob));
+ return Arrays.copyOf(padded, expected.length);
+ });
+ }
+
+ private static ReadPathTimings benchmarkRawDoubleLzmaReadPath(
+ File csvFile, double[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ double[] vals = parseDoublesFromTokens(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ for (int i = 0; i < expected.length; i++) {
+ Assert.assertEquals(
+ Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(vals[i]));
+ }
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ out.avgBinReadNs = sumBr / BENCH_PHASE_REPEATS;
+ Assert.assertNotNull(blob);
+ out.compressedBytesObserved = blob.length;
+
+ double[] decoded = new double[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+
+ t0 = System.nanoTime();
+ double[] d = littleEndianBytesToDoubles(unc);
+ System.arraycopy(d, 0, decoded, 0, d.length);
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+
+ // if (rep == 0) {
+ // for (int i = 0; i < expected.length; i++) {
+ // Assert.assertEquals(
+ // Double.doubleToLongBits(expected[i]), Double.doubleToLongBits(decoded[i]));
+ // }
+ // }
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedDoublesCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+
+ return out;
+ }
+
+ private static ReadPathTimings benchmarkDictionaryReadPath(
+ File csvFile, long[] expected, Path binPath, Path decodedCsvPath) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+
+ ReadPathTimings out = new ReadPathTimings();
+
+ long sumVerify = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ List<String[]> rows = readFullCsvAllColumnsAsStringRows(csvFile);
+ long t1 = System.nanoTime();
+ sumVerify += (t1 - t0);
+ List<String> tokens = firstColumnTokensFromRows(rows);
+ long[] vals = scaleTokensPerSubcolumnBlock(tokens);
+ Assert.assertEquals(expected.length, vals.length);
+ Assert.assertArrayEquals(expected, vals);
+ }
+ out.avgDatasetVerifyReadNs = sumVerify / BENCH_PHASE_REPEATS;
+
+ long sumBr = 0;
+ byte[] blob = null;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ if (benchWindowsBinReadShrinkBeforeReadEnv()) {
+ blob = null;
+ }
+ blob = readCompressedBinBench(binPath);
+ long t1 = System.nanoTime();
+ sumBr += (t1 - t0);
+ }
+ out.avgBinReadNs = sumBr / BENCH_PHASE_REPEATS;
+ Assert.assertNotNull(blob);
+ out.compressedBytesObserved = blob.length;
+
+ long[] decoded = new long[expected.length];
+ long sumUnc = 0;
+ long sumDec = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(blob);
+ long t1 = System.nanoTime();
+ sumUnc += (t1 - t0);
+
+ DictionaryDecoder dictionaryDecoder = new DictionaryDecoder();
+ ByteBuffer db = ByteBuffer.wrap(unc);
+ t0 = System.nanoTime();
+ int i = 0;
+ while (dictionaryDecoder.hasNext(db)) {
+ decoded[i++] = binaryToLong(dictionaryDecoder.readBinary(db));
+ }
+ t1 = System.nanoTime();
+ sumDec += (t1 - t0);
+ // Assert.assertEquals(expected.length, i);
+
+ // if (rep == 0) {
+ // Assert.assertArrayEquals(expected, decoded);
+ // }
+ dictionaryDecoder.reset();
+ }
+ out.avgUncompressNs = sumUnc / BENCH_PHASE_REPEATS;
+ out.avgDecodeNs = sumDec / BENCH_PHASE_REPEATS;
+
+ long sumCsv = 0;
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ Files.deleteIfExists(decodedCsvPath);
+ sumCsv += timeDecodedLongCsvWriteNanos(decodedCsvPath, decoded);
+ }
+ out.avgDecodedCsvWriteNs = sumCsv / BENCH_PHASE_REPEATS;
+
+ return out;
+ }
+
+ private static PhaseTiming benchmarkRawDoubleLzma(double[] values) throws IOException {
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZMA2);
+
+ PhaseTiming t = new PhaseTiming();
+
+ byte[] raw = doublesToLittleEndianRawBytes(values);
+ byte[] compressed0 = compressLzma2BenchPreset(raw);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertRawDoubleRoundtrip(values, back0);
+ t.encodedLen = raw.length;
+ t.compressedLen = compressed0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ long t0 = System.nanoTime();
+ byte[] cmp = compressLzma2BenchPreset(raw);
+ long t1 = System.nanoTime();
+ t.sumCompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(cmp);
+ t1 = System.nanoTime();
+ t.sumUncompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ littleEndianBytesToDoubles(unc);
+ t1 = System.nanoTime();
+ t.sumDecodeNs += (t1 - t0);
+ }
+ return t;
+ }
+
+ private static PhaseTiming benchmarkDictionaryUncompressed(long[] values) throws IOException {
+ ICompressor compressor = ICompressor.getCompressor(CompressionType.UNCOMPRESSED);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.UNCOMPRESSED);
+ PhaseTiming t = new PhaseTiming();
+
+ DictionaryEncoder encoder = new DictionaryEncoder();
+ ByteArrayOutputStream baos0 = new ByteArrayOutputStream();
+ for (long v : values) {
+ encoder.encode(longToPlainBinary(v), baos0);
+ }
+ encoder.flush(baos0);
+ byte[] plain = baos0.toByteArray();
+ byte[] compressed0 = compressor.compress(plain);
+ byte[] back0 = unCompressor.uncompress(compressed0);
+ assertDictionaryRoundtrip(values, back0);
+ t.encodedLen = plain.length;
+ t.compressedLen = compressed0.length;
+
+ for (int rep = 0; rep < BENCH_PHASE_REPEATS; rep++) {
+ DictionaryEncoder enc = new DictionaryEncoder();
+
+ long t0 = System.nanoTime();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (long v : values) {
+ enc.encode(longToPlainBinary(v), baos);
+ }
+ enc.flush(baos);
+ byte[] encBytes = baos.toByteArray();
+ long t1 = System.nanoTime();
+ t.sumEncodeFlushNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] cmp = compressor.compress(encBytes);
+ t1 = System.nanoTime();
+ t.sumCompressNs += (t1 - t0);
+
+ t0 = System.nanoTime();
+ byte[] unc = unCompressor.uncompress(cmp);
+ t1 = System.nanoTime();
+ t.sumUncompressNs += (t1 - t0);
+
+ DictionaryDecoder decoder = new DictionaryDecoder();
+ t0 = System.nanoTime();
+ ByteBuffer db = ByteBuffer.wrap(unc);
+ while (decoder.hasNext(db)) {
+ decoder.readBinary(db);
+ }
+ t1 = System.nanoTime();
+ t.sumDecodeNs += (t1 - t0);
+ decoder.reset();
+ }
+ return t;
+ }
+
+ @Test
+ public void testSyntheticRawDoubleLzmaRoundTripAndTiming() throws IOException {
+ double[] values = {
+ 0.0,
+ 1.0,
+ -1.0,
+ 1234567890123.0,
+ (double) (Long.MAX_VALUE / 4),
+ (double) (Long.MIN_VALUE / 4),
+ 42.0,
+ 42.0,
+ 99.0,
+ -300.0
+ };
+ PhaseTiming t = benchmarkRawDoubleLzma(values);
+ Assert.assertTrue(t.compressedLen <= t.encodedLen + 200);
+ System.out.printf(
+ "RAW_DOUBLE+LZMA2 synthetic n=%d enc+flush_avg_ns=%.1f compress_avg_ns=%.1f "
+ + "uncompress_avg_ns=%.1f decode_avg_ns=%.1f encoded_B=%d compressed_B=%d%n",
+ values.length,
+ t.avgEncodeFlushNs(),
+ t.avgCompressNs(),
+ t.avgUncompressNs(),
+ t.avgDecodeNs(),
+ t.encodedLen,
+ t.compressedLen);
+ }
+
+ @Test
+ public void testSyntheticDictionaryUncompressedRoundTripAndTiming() throws IOException {
+ long[] values = {10L, 20L, 10L, 30L, 20L, 10L};
+ PhaseTiming t = benchmarkDictionaryUncompressed(values);
+ System.out.printf(
+ "DICTIONARY+UNC synthetic n=%d enc+flush_avg_ns=%.1f compress_avg_ns=%.1f "
+ + "uncompress_avg_ns=%.1f decode_avg_ns=%.1f encoded_B=%d compressed_B=%d%n",
+ values.length,
+ t.avgEncodeFlushNs(),
+ t.avgCompressNs(),
+ t.avgUncompressNs(),
+ t.avgDecodeNs(),
+ t.encodedLen,
+ t.compressedLen);
+ }
+
+ @Test
+ public void testHddEncodeCompressWriteBinCsvIfPresent() throws IOException {
+ runEncodeCompressWriteBinCsv(HDD_PROFILE);
+ }
+
+ @Test
+ public void testHddDecodeBinWriteDecodedCsvIfPresent() throws IOException {
+ runDecodeBinWriteDecodedCsv(HDD_PROFILE);
+ }
+
+ @Test
+ public void testSsdEncodeCompressWriteBinCsvIfPresent() throws IOException {
+ runEncodeCompressWriteBinCsv(SSD_PROFILE);
+ }
+
+ @Test
+ public void testSsdDecodeBinWriteDecodedCsvIfPresent() throws IOException {
+ runDecodeBinWriteDecodedCsv(SSD_PROFILE);
+ }
+
+ private static void runEncodeCompressWriteBinCsv(BenchStorageProfile profile) throws IOException {
+ File sourceDir = new File(profile.datasetDir);
+ Assume.assumeTrue(
+ "Skip when CSV dir missing: " + profile.datasetDir, sourceDir.isDirectory());
+ File[] csvFiles = listBenchmarkDatasetCsvFiles(sourceDir);
+ Assume.assumeTrue(
+ "Skip when no CSV files in " + profile.datasetDir, csvFiles.length > 0);
+
+ Path binsDir = Paths.get(profile.binOutputDir);
+ Path decodedRoot = Paths.get(profile.decodedCsvDir);
+ ensureDirectory(binsDir);
+ ensureDirectory(decodedRoot);
+
+ Path writeCsvPath = Paths.get(profile.writeMetricsCsvPath);
+ Path manifestPath = Paths.get(profile.compressManifestCsvPath);
+ ensureDirectory(writeCsvPath.getParent());
+ ensureDirectory(manifestPath.getParent());
+
+ try (BufferedWriter writeMetrics =
+ Files.newBufferedWriter(
+ writeCsvPath,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE);
+ BufferedWriter manifest =
+ Files.newBufferedWriter(
+ manifestPath,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE)) {
+
+ writeMetrics.write(
+ "Dataset,Encoding Algorithm,Write Total Time Nanos,Dataset Read "
+ + "Time Nanos,Write CPU Time Nanos,Write IO Time Nanos,Points,Max "
+ + "Decimal Precision,Multiplier,TsFile Size Bytes,Write Encode Loop Time "
+ + "Nanos,Write Encode Flush Time Nanos,Write Compress Time Nanos\n");
+ manifest.write(
+ "Dataset,Encoding Algorithm,Source CSV Path,Points,Encoded "
+ + "Uncompressed Bytes,Raw Plain Codec\n");
+
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] encode-start | profile=%s | sourceDir=%s | datasets=%d | binsDir=%s%n",
+ profile.label,
+ sourceDir.getAbsolutePath(),
+ csvFiles.length,
+ binsDir.toAbsolutePath());
+ System.out.flush();
+
+ for (File f : csvFiles) {
+ List<String> tokens = readFirstColumnTokens(f);
+ Assume.assumeFalse(tokens.isEmpty());
+ int maxPrecAll = maxDecimalPrecision(tokens);
+ double[] dValues = parseDoublesFromTokens(tokens);
+ long[] dictValues = scaleTokensPerSubcolumnBlock(tokens);
+ Assert.assertEquals(dValues.length, dictValues.length);
+ String datasetName = stripCsvExtension(f.getName());
+ String srcPath = f.getCanonicalPath().replace('\\', '/');
+
+ long avgReadFullCsvNs = averageCsvFullReadWithoutNumericParseNanos(f);
+ int boundedPrec = Math.min(maxPrecAll, K_MAX_DECIMAL_PRECISION);
+ long multiplier = multiplierForBoundedPrecision(maxPrecAll);
+
+ int[] intScaled = scaleDoublesToIntsMicroStyle(dValues, maxPrecAll);
+ long[] hbpValues = scaleDoublesToLongsFileWide(dValues, maxPrecAll);
+ long[] rleValues = hbpValues;
+
+ logBenchDatasetHeader("encode", datasetName, f, dValues.length);
+
+ Path binLzma = binsDir.resolve(datasetName + "_" + SUFFIX_PLAIN_LZMA + ".bin");
+ logBenchWarmupDiscarded("EncodeCompressWriteBinCsv", datasetName, ALGO_LZMA_CSV);
+ benchmarkRawDoubleLzmaWritePath(dValues, binLzma);
+ logBenchProgress("encode", datasetName, ALGO_LZMA_CSV, dValues.length, binLzma);
+ WritePathTimings wLz = benchmarkRawDoubleLzmaWritePath(dValues, binLzma);
+ wLz.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_LZMA_CSV, wLz, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_LZMA_CSV,
+ srcPath,
+ dValues.length,
+ wLz.encodedUncompressedBytes,
+ 1);
+
+ Path binDict = binsDir.resolve(datasetName + "_" + SUFFIX_DICTIONARY + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_DICTIONARY_CSV, dictValues.length, binDict);
+ WritePathTimings wDc = benchmarkDictionaryWritePath(dictValues, binDict);
+ wDc.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_DICTIONARY_CSV,
+ wDc,
+ dictValues.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_DICTIONARY_CSV,
+ srcPath,
+ dictValues.length,
+ wDc.encodedUncompressedBytes,
+ 0);
+
+ Path binGorilla = binsDir.resolve(datasetName + "_" + SUFFIX_GORILLA + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_GORILLA_CSV, dValues.length, binGorilla);
+ WritePathTimings wGorilla = benchmarkGorillaWritePath(dValues, binGorilla);
+ wGorilla.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_GORILLA_CSV, wGorilla, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_GORILLA_CSV,
+ srcPath,
+ dValues.length,
+ wGorilla.encodedUncompressedBytes,
+ 0);
+
+ Path binChimp = binsDir.resolve(datasetName + "_" + SUFFIX_CHIMP + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_CHIMP_CSV, dValues.length, binChimp);
+ WritePathTimings wChimp = benchmarkChimpWritePath(dValues, binChimp);
+ wChimp.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_CHIMP_CSV, wChimp, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_CHIMP_CSV,
+ srcPath,
+ dValues.length,
+ wChimp.encodedUncompressedBytes,
+ 0);
+
+ Path binElf = binsDir.resolve(datasetName + "_" + SUFFIX_ELF + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_ELF_CSV, dValues.length, binElf);
+ WritePathTimings wElf = benchmarkElfWritePath(dValues, binElf);
+ wElf.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_ELF_CSV, wElf, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_ELF_CSV,
+ srcPath,
+ dValues.length,
+ wElf.encodedUncompressedBytes,
+ 0);
+
+ Path binBuff = binsDir.resolve(datasetName + "_" + SUFFIX_BUFF + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_BUFF_CSV, dValues.length, binBuff);
+ WritePathTimings wBuff = benchmarkBuffWritePath(dValues, maxPrecAll, binBuff);
+ wBuff.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_BUFF_CSV, wBuff, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_BUFF_CSV,
+ srcPath,
+ dValues.length,
+ wBuff.encodedUncompressedBytes,
+ 0);
+
+ Path binAlp = binsDir.resolve(datasetName + "_" + SUFFIX_ALP + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_ALP_CSV, dValues.length, binAlp);
+ WritePathTimings wAlp = benchmarkAlpWritePath(dValues, maxPrecAll, binAlp);
+ wAlp.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics, datasetName, ALGO_ALP_CSV, wAlp, dValues.length, boundedPrec, multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_ALP_CSV,
+ srcPath,
+ dValues.length,
+ wAlp.encodedUncompressedBytes,
+ 0);
+
+ Path binBitweaving = binsDir.resolve(datasetName + "_" + SUFFIX_BITWEAVING + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_BITWEAVING_CSV, hbpValues.length, binBitweaving);
+ WritePathTimings wHbp = benchmarkHbpWritePath(hbpValues, binBitweaving);
+ wHbp.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_BITWEAVING_CSV,
+ wHbp,
+ hbpValues.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_BITWEAVING_CSV,
+ srcPath,
+ hbpValues.length,
+ wHbp.encodedUncompressedBytes,
+ 0);
+
+ Path binS8 = binsDir.resolve(datasetName + "_" + SUFFIX_SIMPLE8B + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_SIMPLE8B_CSV, intScaled.length, binS8);
+ WritePathTimings wS8 = benchmarkSimple8bWritePath(intScaled, binS8);
+ wS8.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_SIMPLE8B_CSV,
+ wS8,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_SIMPLE8B_CSV,
+ srcPath,
+ intScaled.length,
+ wS8.encodedUncompressedBytes,
+ 0);
+
+ Path binFp = binsDir.resolve(datasetName + "_" + SUFFIX_FASTPFOR + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_FASTPFOR_CSV, intScaled.length, binFp);
+ WritePathTimings wFp = benchmarkFastPforWritePath(intScaled, binFp);
+ wFp.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_FASTPFOR_CSV,
+ wFp,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_FASTPFOR_CSV,
+ srcPath,
+ intScaled.length,
+ wFp.encodedUncompressedBytes,
+ 0);
+
+ Path binRle = binsDir.resolve(datasetName + "_" + SUFFIX_RLE + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_RLE_CSV, rleValues.length, binRle);
+ WritePathTimings wRle = benchmarkRleWritePath(rleValues, binRle);
+ wRle.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_RLE_CSV,
+ wRle,
+ rleValues.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_RLE_CSV,
+ srcPath,
+ rleValues.length,
+ wRle.encodedUncompressedBytes,
+ 0);
+
+ Path binBitPacking = binsDir.resolve(datasetName + "_" + SUFFIX_BITPACKING + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_BITPACKING_CSV, intScaled.length, binBitPacking);
+ WritePathTimings wBp = benchmarkBitPackingWritePath(intScaled, binBitPacking);
+ wBp.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_BITPACKING_CSV,
+ wBp,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_BITPACKING_CSV,
+ srcPath,
+ intScaled.length,
+ wBp.encodedUncompressedBytes,
+ 0);
+
+ Path binSprintz = binsDir.resolve(datasetName + "_" + SUFFIX_SPRINTZ + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_SPRINTZ_CSV, intScaled.length, binSprintz);
+ WritePathTimings wSprintz = benchmarkSprintzWritePath(intScaled, binSprintz);
+ wSprintz.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_SPRINTZ_CSV,
+ wSprintz,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_SPRINTZ_CSV,
+ srcPath,
+ intScaled.length,
+ wSprintz.encodedUncompressedBytes,
+ 0);
+
+ Path binTs2Diff = binsDir.resolve(datasetName + "_" + SUFFIX_TS_2DIFF + ".bin");
+ logBenchProgress("encode", datasetName, ALGO_TS_2DIFF_CSV, intScaled.length, binTs2Diff);
+ WritePathTimings wTs2Diff = benchmarkTs2DiffWritePath(intScaled, binTs2Diff);
+ wTs2Diff.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_TS_2DIFF_CSV,
+ wTs2Diff,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_TS_2DIFF_CSV,
+ srcPath,
+ intScaled.length,
+ wTs2Diff.encodedUncompressedBytes,
+ 0);
+
+ Path binSubcolumn = binsDir.resolve(datasetName + "_" + SUFFIX_SUBCOLUMN + ".bin");
+ logBenchProgress(
+ "encode", datasetName, ALGO_SUBCOLUMN_CSV, intScaled.length, binSubcolumn);
+ WritePathTimings wSubcolumn = benchmarkSubcolumnWritePath(intScaled, binSubcolumn);
+ wSubcolumn.avgDatasetReadNs = avgReadFullCsvNs;
+ writeEncodeMetricsRow(
+ writeMetrics,
+ datasetName,
+ ALGO_SUBCOLUMN_CSV,
+ wSubcolumn,
+ intScaled.length,
+ boundedPrec,
+ multiplier);
+ writeManifestRow(
+ manifest,
+ datasetName,
+ ALGO_SUBCOLUMN_CSV,
+ srcPath,
+ intScaled.length,
+ wSubcolumn.encodedUncompressedBytes,
+ 0);
+
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] encode-done | profile=%s | dataset=%s | algorithms=%d%n",
+ profile.label,
+ datasetName,
+ BENCH_ALGORITHM_COUNT);
+ System.out.flush();
+ }
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] encode-finish | profile=%s | writeMetrics=%s | manifest=%s%n",
+ profile.label,
+ writeCsvPath.toAbsolutePath(),
+ manifestPath.toAbsolutePath());
+ System.out.flush();
+ }
+ }
+
+ private static void runDecodeBinWriteDecodedCsv(BenchStorageProfile profile) throws IOException {
+ File sourceDir = new File(profile.datasetDir);
+ Assume.assumeTrue(
+ "Skip when CSV dir missing: " + profile.datasetDir, sourceDir.isDirectory());
+ File[] csvFiles = listBenchmarkDatasetCsvFiles(sourceDir);
+ Assume.assumeTrue(
+ "Skip when no CSV files in " + profile.datasetDir, csvFiles.length > 0);
+
+ Path binsDir = Paths.get(profile.binOutputDir);
+ Path decodedRoot = Paths.get(profile.decodedCsvDir);
+ Path readCsvPath = Paths.get(profile.readMetricsCsvPath);
+ Path manifestPath = Paths.get(profile.compressManifestCsvPath);
+
+ ensureDirectory(decodedRoot);
+ ensureDirectory(readCsvPath.getParent());
+
+ Map<String, Map<String, ManifestRecord>> manifestMap = loadManifestRows(manifestPath);
+
+ try (BufferedWriter readMetrics =
+ Files.newBufferedWriter(
+ readCsvPath,
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.TRUNCATE_EXISTING,
+ StandardOpenOption.WRITE)) {
+
+ readMetrics.write(
+ "Dataset,Encoding Algorithm,Read Total Time Nanos,Read CPU Time "
+ + "Nanos,Dataset Verify Read Time Nanos,Read IO Time Nanos,Decoded CSV "
+ + "Write Time Nanos,Points,"
+ + "TsFile Size Bytes\n");
+
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] decode-start | profile=%s | sourceDir=%s | datasets=%d | binsDir=%s%n",
+ profile.label,
+ sourceDir.getAbsolutePath(),
+ csvFiles.length,
+ binsDir.toAbsolutePath());
+ System.out.flush();
+
+ for (File f : csvFiles) {
+ List<String> tokens = readFirstColumnTokens(f);
+ Assume.assumeFalse(tokens.isEmpty());
+ double[] expectedDoubles = parseDoublesFromTokens(tokens);
+ long[] dictValues = scaleTokensPerSubcolumnBlock(tokens);
+ int maxPrecAll = maxDecimalPrecision(tokens);
+ int[] intScaled = scaleDoublesToIntsMicroStyle(expectedDoubles, maxPrecAll);
+ long[] hbpValues = scaleDoublesToLongsFileWide(expectedDoubles, maxPrecAll);
+ long[] rleValues = hbpValues;
+ Assert.assertEquals(expectedDoubles.length, dictValues.length);
+ String datasetName = stripCsvExtension(f.getName());
+
+ Map<String, ManifestRecord> rows = manifestMap.get(datasetName);
+ Assert.assertNotNull(
+ "No manifest rows for dataset "
+ + datasetName
+ + " (run "
+ + (profile.label.equals("HDD")
+ ? "testHddEncodeCompressWriteBinCsvIfPresent"
+ : "testSsdEncodeCompressWriteBinCsvIfPresent")
+ + " first)",
+ rows);
+
+ logBenchDatasetHeader("decode", datasetName, f, expectedDoubles.length);
+
+ ManifestRecord lzRec = rows.get(ALGO_LZMA_CSV);
+ Assert.assertNotNull("Missing LZMA manifest row for " + datasetName, lzRec);
+ Assert.assertEquals(expectedDoubles.length, lzRec.points);
+ Assert.assertEquals(1, lzRec.rawPlainCodec);
+ Path binLzma = binsDir.resolve(datasetName + "_" + SUFFIX_PLAIN_LZMA + ".bin");
+ Assert.assertTrue("Missing bin: " + binLzma, Files.isRegularFile(binLzma));
+ Path decodedLzma = decodedRoot.resolve(datasetName + "_" + SUFFIX_PLAIN_LZMA + "_decoded.csv");
+ logBenchWarmupDiscarded("DecodeBinWriteDecodedCsv", datasetName, ALGO_LZMA_CSV);
+ benchmarkRawDoubleLzmaReadPath(f, expectedDoubles, binLzma, decodedLzma);
+ logBenchProgress("decode", datasetName, ALGO_LZMA_CSV, expectedDoubles.length, binLzma);
+ ReadPathTimings rLz =
+ benchmarkRawDoubleLzmaReadPath(f, expectedDoubles, binLzma, decodedLzma);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_LZMA_CSV, rLz, expectedDoubles.length);
+
+ ManifestRecord dcRec = rows.get(ALGO_DICTIONARY_CSV);
+ Assert.assertNotNull("Missing DICTIONARY manifest row for " + datasetName, dcRec);
+ Path binDict = binsDir.resolve(datasetName + "_" + SUFFIX_DICTIONARY + ".bin");
+ Assert.assertTrue("Missing bin: " + binDict, Files.isRegularFile(binDict));
+ Path decodedDict = decodedRoot.resolve(datasetName + "_" + SUFFIX_DICTIONARY + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_DICTIONARY_CSV, dictValues.length, binDict);
+ ReadPathTimings rDc =
+ benchmarkDictionaryReadPath(f, dictValues, binDict, decodedDict);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_DICTIONARY_CSV, rDc, dictValues.length);
+
+ Assert.assertNotNull(rows.get(ALGO_GORILLA_CSV));
+ Path binGorilla = binsDir.resolve(datasetName + "_" + SUFFIX_GORILLA + ".bin");
+ Assert.assertTrue("Missing bin: " + binGorilla, Files.isRegularFile(binGorilla));
+ Path decodedGorilla = decodedRoot.resolve(datasetName + "_" + SUFFIX_GORILLA + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_GORILLA_CSV, expectedDoubles.length, binGorilla);
+ ReadPathTimings rGorilla =
+ benchmarkGorillaReadPath(f, expectedDoubles, binGorilla, decodedGorilla);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_GORILLA_CSV, rGorilla, expectedDoubles.length);
+
+ Assert.assertNotNull(rows.get(ALGO_CHIMP_CSV));
+ Path binChimp = binsDir.resolve(datasetName + "_" + SUFFIX_CHIMP + ".bin");
+ Assert.assertTrue("Missing bin: " + binChimp, Files.isRegularFile(binChimp));
+ Path decodedChimp = decodedRoot.resolve(datasetName + "_" + SUFFIX_CHIMP + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_CHIMP_CSV, expectedDoubles.length, binChimp);
+ ReadPathTimings rChimp =
+ benchmarkChimpReadPath(f, expectedDoubles, binChimp, decodedChimp);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_CHIMP_CSV, rChimp, expectedDoubles.length);
+
+ Assert.assertNotNull(rows.get(ALGO_ELF_CSV));
+ Path binElf = binsDir.resolve(datasetName + "_" + SUFFIX_ELF + ".bin");
+ Assert.assertTrue("Missing bin: " + binElf, Files.isRegularFile(binElf));
+ Path decodedElf = decodedRoot.resolve(datasetName + "_" + SUFFIX_ELF + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_ELF_CSV, expectedDoubles.length, binElf);
+ ReadPathTimings rElf =
+ benchmarkElfReadPath(f, expectedDoubles, binElf, decodedElf);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_ELF_CSV, rElf, expectedDoubles.length);
+
+ Assert.assertNotNull(rows.get(ALGO_BUFF_CSV));
+ Path binBuff = binsDir.resolve(datasetName + "_" + SUFFIX_BUFF + ".bin");
+ Assert.assertTrue("Missing bin: " + binBuff, Files.isRegularFile(binBuff));
+ Path decodedBuff = decodedRoot.resolve(datasetName + "_" + SUFFIX_BUFF + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_BUFF_CSV, expectedDoubles.length, binBuff);
+ ReadPathTimings rBuff =
+ benchmarkBuffReadPath(f, expectedDoubles, maxPrecAll, binBuff, decodedBuff);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_BUFF_CSV, rBuff, expectedDoubles.length);
+
+ Assert.assertNotNull(rows.get(ALGO_ALP_CSV));
+ Path binAlp = binsDir.resolve(datasetName + "_" + SUFFIX_ALP + ".bin");
+ Assert.assertTrue("Missing bin: " + binAlp, Files.isRegularFile(binAlp));
+ Path decodedAlp = decodedRoot.resolve(datasetName + "_" + SUFFIX_ALP + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_ALP_CSV, expectedDoubles.length, binAlp);
+ ReadPathTimings rAlp =
+ benchmarkAlpReadPath(f, expectedDoubles, maxPrecAll, binAlp, decodedAlp);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_ALP_CSV, rAlp, expectedDoubles.length);
+
+ Assert.assertNotNull(rows.get(ALGO_BITWEAVING_CSV));
+ Path binHbp = binsDir.resolve(datasetName + "_" + SUFFIX_BITWEAVING + ".bin");
+ Assert.assertTrue("Missing bin: " + binHbp, Files.isRegularFile(binHbp));
+ Path decodedBitweaving =
+ decodedRoot.resolve(datasetName + "_" + SUFFIX_BITWEAVING + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_BITWEAVING_CSV, hbpValues.length, binHbp);
+ ReadPathTimings rHbp =
+ benchmarkHbpReadPath(f, hbpValues, binHbp, decodedBitweaving);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_BITWEAVING_CSV, rHbp, hbpValues.length);
+
+ Assert.assertNotNull(rows.get(ALGO_SIMPLE8B_CSV));
+ Path binS8 = binsDir.resolve(datasetName + "_" + SUFFIX_SIMPLE8B + ".bin");
+ Assert.assertTrue("Missing bin: " + binS8, Files.isRegularFile(binS8));
+ Path decodedS8 = decodedRoot.resolve(datasetName + "_" + SUFFIX_SIMPLE8B + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_SIMPLE8B_CSV, intScaled.length, binS8);
+ ReadPathTimings rS8 =
+ benchmarkSimple8bReadPath(f, intScaled, binS8, decodedS8);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_SIMPLE8B_CSV, rS8, intScaled.length);
+
+ Assert.assertNotNull(rows.get(ALGO_FASTPFOR_CSV));
+ Path binFp = binsDir.resolve(datasetName + "_" + SUFFIX_FASTPFOR + ".bin");
+ Assert.assertTrue("Missing bin: " + binFp, Files.isRegularFile(binFp));
+ Path decodedFp = decodedRoot.resolve(datasetName + "_" + SUFFIX_FASTPFOR + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_FASTPFOR_CSV, intScaled.length, binFp);
+ ReadPathTimings rFp =
+ benchmarkFastPforReadPath(f, intScaled, binFp, decodedFp);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_FASTPFOR_CSV, rFp, intScaled.length);
+
+ Assert.assertNotNull(rows.get(ALGO_RLE_CSV));
+ Path binRle = binsDir.resolve(datasetName + "_" + SUFFIX_RLE + ".bin");
+ Assert.assertTrue("Missing bin: " + binRle, Files.isRegularFile(binRle));
+ Path decodedRle = decodedRoot.resolve(datasetName + "_" + SUFFIX_RLE + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_RLE_CSV, rleValues.length, binRle);
+ ReadPathTimings rRle = benchmarkRleReadPath(f, rleValues, binRle, decodedRle);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_RLE_CSV, rRle, rleValues.length);
+
+ Assert.assertNotNull(rows.get(ALGO_BITPACKING_CSV));
+ Path binBitPacking = binsDir.resolve(datasetName + "_" + SUFFIX_BITPACKING + ".bin");
+ Assert.assertTrue("Missing bin: " + binBitPacking, Files.isRegularFile(binBitPacking));
+ Path decodedBitPacking =
+ decodedRoot.resolve(datasetName + "_" + SUFFIX_BITPACKING + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_BITPACKING_CSV, intScaled.length, binBitPacking);
+ ReadPathTimings rBp =
+ benchmarkBitPackingReadPath(f, intScaled, binBitPacking, decodedBitPacking);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_BITPACKING_CSV, rBp, intScaled.length);
+
+ Assert.assertNotNull(rows.get(ALGO_SPRINTZ_CSV));
+ Path binSprintz = binsDir.resolve(datasetName + "_" + SUFFIX_SPRINTZ + ".bin");
+ Assert.assertTrue("Missing bin: " + binSprintz, Files.isRegularFile(binSprintz));
+ Path decodedSprintz = decodedRoot.resolve(datasetName + "_" + SUFFIX_SPRINTZ + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_SPRINTZ_CSV, intScaled.length, binSprintz);
+ ReadPathTimings rSprintz =
+ benchmarkSprintzReadPath(f, intScaled, binSprintz, decodedSprintz);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_SPRINTZ_CSV, rSprintz, intScaled.length);
+
+ Assert.assertNotNull(rows.get(ALGO_TS_2DIFF_CSV));
+ Path binTs2Diff = binsDir.resolve(datasetName + "_" + SUFFIX_TS_2DIFF + ".bin");
+ Assert.assertTrue("Missing bin: " + binTs2Diff, Files.isRegularFile(binTs2Diff));
+ Path decodedTs2Diff =
+ decodedRoot.resolve(datasetName + "_" + SUFFIX_TS_2DIFF + "_decoded.csv");
+ logBenchProgress("decode", datasetName, ALGO_TS_2DIFF_CSV, intScaled.length, binTs2Diff);
+ ReadPathTimings rTs2Diff =
+ benchmarkTs2DiffReadPath(f, intScaled, binTs2Diff, decodedTs2Diff);
+ writeReadMetricsRow(readMetrics, datasetName, ALGO_TS_2DIFF_CSV, rTs2Diff, intScaled.length);
+
+ Assert.assertNotNull(rows.get(ALGO_SUBCOLUMN_CSV));
+ Path binSubcolumn = binsDir.resolve(datasetName + "_" + SUFFIX_SUBCOLUMN + ".bin");
+ Assert.assertTrue("Missing bin: " + binSubcolumn, Files.isRegularFile(binSubcolumn));
+ Path decodedSubcolumn =
+ decodedRoot.resolve(datasetName + "_" + SUFFIX_SUBCOLUMN + "_decoded.csv");
+ logBenchProgress(
+ "decode", datasetName, ALGO_SUBCOLUMN_CSV, intScaled.length, binSubcolumn);
+ ReadPathTimings rSubcolumn =
+ benchmarkSubcolumnReadPath(f, intScaled, binSubcolumn, decodedSubcolumn);
+ writeReadMetricsRow(
+ readMetrics, datasetName, ALGO_SUBCOLUMN_CSV, rSubcolumn, intScaled.length);
+
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] decode-done | profile=%s | dataset=%s | algorithms=%d%n",
+ profile.label,
+ datasetName,
+ BENCH_ALGORITHM_COUNT);
+ System.out.flush();
+ }
+ System.out.printf(
+ "[DatasetEncoderCompressBinBench] decode-finish | profile=%s | readMetrics=%s%n",
+ profile.label,
+ readCsvPath.toAbsolutePath());
+ System.out.flush();
+ }
+ }
+
+ private static String stripCsvExtension(String filename) {
+ if (filename.endsWith(".csv")) {
+ return filename.substring(0, filename.length() - 4);
+ }
+ return filename;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaLongTest.java
new file mode 100644
index 0000000..49f255c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaLongTest.java
@@ -0,0 +1,717 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class DeltaLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end - 1; j++) {
+ ts_block_delta[j - base] = ts_block[j + 1] - ts_block[j];
+
+ if (ts_block_delta[j - base] < value_delta_min) {
+ value_delta_min = ts_block_delta[j - base];
+ }
+ if (ts_block_delta[j - base] > value_delta_max) {
+ value_delta_max = ts_block_delta[j - base];
+ }
+ }
+
+ for (int j = 0; j < remaining - 1; j++) {
+ ts_block_delta[j] -= value_delta_min;
+ }
+
+ min_delta[0] = ts_block[base];
+ min_delta[1] = value_delta_max;
+ min_delta[2] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long first_value = data[block_index * block_size];
+ first_value = SPRINTZBPLongTest.zigzag(first_value);
+
+ long2Bytes(first_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ long min_delta_value = SPRINTZBPLongTest.zigzag(min_delta[2]);
+ min_delta_value = SPRINTZBPLongTest.zigzag(min_delta_value);
+
+ long2Bytes(min_delta_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(min_delta[1] - min_delta[2]);
+
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ encode_pos = bitPacking(data_delta, bw, encode_pos, encoded_result, remainder - 1);
+
+ // encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ // encoded_result, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ min_delta[0] = SPRINTZBPLongTest.deZigzag(min_delta[0]);
+ encode_pos += 8;
+
+ min_delta[2] = bytes2Long(encoded_result, encode_pos, 8);
+ min_delta[2] = SPRINTZBPLongTest.deZigzag(min_delta[2]);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long[] block_data = new long[remainder - 1];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, remainder - 1, block_data);
+
+ data[block_index * block_size] = min_delta[0];
+
+ for (int i = 1; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i - 1] + data[block_index * block_size + i - 1] + min_delta[2];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "delta_long.csv";
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaSubcolumnLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaSubcolumnLongTest.java
new file mode 100644
index 0000000..0563cb1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DeltaSubcolumnLongTest.java
@@ -0,0 +1,728 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class DeltaSubcolumnLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end - 1; j++) {
+ ts_block_delta[j - base] = ts_block[j + 1] - ts_block[j];
+
+ if (ts_block_delta[j - base] < value_delta_min) {
+ value_delta_min = ts_block_delta[j - base];
+ }
+ if (ts_block_delta[j - base] > value_delta_max) {
+ value_delta_max = ts_block_delta[j - base];
+ }
+ }
+
+ for (int j = 0; j < remaining - 1; j++) {
+ ts_block_delta[j] -= value_delta_min;
+ }
+
+ min_delta[0] = ts_block[base];
+ min_delta[1] = value_delta_max;
+ min_delta[2] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long first_value = min_delta[0];
+ first_value = SPRINTZBPLongTest.zigzag(first_value);
+
+ long2Bytes(first_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ long min_delta_value = SPRINTZBPLongTest.zigzag(min_delta[2]);
+ min_delta_value = SPRINTZBPLongTest.zigzag(min_delta_value);
+
+ long2Bytes(min_delta_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(min_delta[1] - min_delta[2]);
+
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int m = SubcolumnLongTest.bitWidth(min_delta[1] - min_delta[2]);
+
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder - 1, m, block_size);
+ }
+
+ encode_pos = SubcolumnLongTest.SubcolumnEncoder(data_delta, encode_pos, encoded_result, beta, block_size);
+
+ // encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ // encoded_result, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ min_delta[0] = SPRINTZBPLongTest.deZigzag(min_delta[0]);
+ encode_pos += 8;
+
+ min_delta[2] = bytes2Long(encoded_result, encode_pos, 8);
+ min_delta[2] = SPRINTZBPLongTest.deZigzag(min_delta[2]);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long[] block_data = new long[remainder - 1];
+
+ encode_pos = SubcolumnLongTest.SubcolumnDecoder(encoded_result, encode_pos, block_data, block_size);
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, remainder - 1, block_data);
+
+ data[block_index * block_size] = min_delta[0];
+
+ for (int i = 1; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i - 1] + data[block_index * block_size + i - 1] + min_delta[2];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 3;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "delta_subcolumn_long.csv";
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongOnSortedTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongOnSortedTest.java
new file mode 100644
index 0000000..fc4122b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongOnSortedTest.java
@@ -0,0 +1,827 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class DictionaryLongOnSortedTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ Map<Long, Integer> valueToIndex,
+ List<Long> dictionary) {
+ int[] encodedIndices = new int[remaining];
+
+ int dictIndex = 0;
+
+ for (int j = 0; j < remaining; j++) {
+ long value = ts_block[i * block_size + j];
+
+ if (!valueToIndex.containsKey(value)) {
+ valueToIndex.put(value, dictIndex);
+ dictionary.add(value);
+ encodedIndices[j] = dictIndex;
+ dictIndex++;
+ } else {
+ encodedIndices[j] = valueToIndex.get(value);
+ }
+ }
+
+ return encodedIndices;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+ Map<Long, Integer> valueToIndex = new LinkedHashMap<>();
+ List<Long> dictionary = new ArrayList<>();
+
+ long[] min_delta = new long[3];
+
+ int[] encodedIndices = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta, valueToIndex, dictionary);
+
+ int dictSize = dictionary.size();
+ int2Bytes(dictSize, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ long[] dictArray = new long[dictSize];
+
+ long min_dict_value = Long.MAX_VALUE;
+ long max_dict_value = Long.MIN_VALUE;
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictionary.get(i);
+ if (dictArray[i] < min_dict_value) {
+ min_dict_value = dictArray[i];
+ }
+ if (dictArray[i] > max_dict_value) {
+ max_dict_value = dictArray[i];
+ }
+ }
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictArray[i] - min_dict_value;
+ }
+
+ long2Bytes(min_dict_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_dict_value - min_dict_value);
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ encode_pos = bitPacking(dictArray, bw, encode_pos, encoded_result, dictSize);
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ encode_pos = bitPacking(encodedIndices, indexBitWidth, encode_pos, encoded_result, remainder);
+
+ // encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ // encoded_result, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ int dictSize = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long min_dict_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int dictBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long[] dictionary = new long[dictSize];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dictBitWidth, dictSize, dictionary);
+
+ for (int i = 0; i < dictSize; i++) {
+ dictionary[i] = dictionary[i] + min_dict_value;
+ }
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int[] encodedIndices = new int[remainder];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth, remainder, encodedIndices);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = dictionary[encodedIndices[i]];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// // String output_parent_dir = parent_dir + "result/";
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+// // String output_parent_dir = parent_dir + "result/";
+//
+// String outputPath = output_parent_dir + "dictionary_long.csv";
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/"; //""D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "dictionary_long_on_sorted.csv";
+
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Points",
+ "Encoding Time",
+ "Decoding Time",
+ "Compressed Size",
+ "Compression Ratio",
+ "Encoding Time Sort",
+ "Decoding Time Sort",
+ "Compressed Size Sort",
+ "Compression Ratio Sort"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+ long[] data1_arr = new long[data1.size()];
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ data1_arr[i] = i;
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 17];
+ byte[] encoded_result1 = new byte[data2_arr.length * 17];
+
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int length1 = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(data1_arr, block_size, encoded_result1);
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data1_arr_decoded = new long[data2_arr.length];
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data1_arr_decoded = Decoder(encoded_result1);
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ Integer[] indices = new Integer[data1_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = i;
+ }
+ long[] finalData2_arr = data2_arr;
+ Arrays.sort(indices, (i, j) -> Long.compare(finalData2_arr[i], finalData2_arr[j]));
+ long[] sortedData1 = new long[data1_arr.length];
+ long[] sortedData2 = new long[data2_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ sortedData1[i] = data1_arr[indices[i]];
+ sortedData2[i] = data2_arr[indices[i]];
+ }
+
+ System.out.println(max_decimal);
+ encoded_result = new byte[data2_arr.length * 8];
+ encoded_result1 = new byte[data2_arr.length * 8];
+
+ long encodeTime_sort = 0;
+ long decodeTime_sort = 0;
+ double ratio_sort = 0;
+ double compressed_size_sort = 0;
+
+ int length_sort = 0;
+ int length1_sort = 0;
+
+ long s_sort = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(sortedData1, block_size, encoded_result1);
+ length = Encoder(sortedData2, block_size, encoded_result);
+ }
+
+ long e_sort = System.nanoTime();
+ encodeTime_sort += ((e_sort - s_sort) / repeatTime);
+ length += length1;
+ compressed_size_sort += length;
+
+ double ratioTmp_sort;
+
+ ratioTmp_sort = compressed_size_sort / (double) (data1.size() * Long.BYTES*2);
+
+ ratio_sort += ratioTmp_sort;
+
+ System.out.println("Decode");
+
+ long[] data1_arr_decoded_sort = new long[data2_arr.length];
+ long[] data2_arr_decoded_sort = new long[data2_arr.length];
+
+ s_sort = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data1_arr_decoded_sort = Decoder(encoded_result1);
+ data2_arr_decoded_sort = Decoder(encoded_result);
+ }
+
+ e_sort = System.nanoTime();
+ decodeTime_sort += ((e_sort - s_sort) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded_sort.length; i++) {
+ assertEquals(sortedData2[i], data2_arr_decoded_sort[i]);
+ }
+ String[] record = {
+ datasetName,
+ "Dictionary",
+ String.valueOf(data1.size()),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio),
+ String.valueOf(encodeTime_sort),
+ String.valueOf(decodeTime_sort),
+ String.valueOf(compressed_size_sort),
+ String.valueOf(ratio_sort),
+ };
+
+// String[] record = {
+// datasetName,
+// "Sub-columns",
+// String.valueOf(encodeTime),
+// String.valueOf(decodeTime),
+// String.valueOf(data1.size()),
+// String.valueOf(compressed_size),
+// String.valueOf(ratio)
+// };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongTest.java
new file mode 100644
index 0000000..910e2f9
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryLongTest.java
@@ -0,0 +1,700 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class DictionaryLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ Map<Long, Integer> valueToIndex,
+ List<Long> dictionary) {
+ int[] encodedIndices = new int[remaining];
+
+ int dictIndex = 0;
+
+ for (int j = 0; j < remaining; j++) {
+ long value = ts_block[i * block_size + j];
+
+ if (!valueToIndex.containsKey(value)) {
+ valueToIndex.put(value, dictIndex);
+ dictionary.add(value);
+ encodedIndices[j] = dictIndex;
+ dictIndex++;
+ } else {
+ encodedIndices[j] = valueToIndex.get(value);
+ }
+ }
+
+ return encodedIndices;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+ Map<Long, Integer> valueToIndex = new LinkedHashMap<>();
+ List<Long> dictionary = new ArrayList<>();
+
+ long[] min_delta = new long[3];
+
+ int[] encodedIndices = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta, valueToIndex, dictionary);
+
+ int dictSize = dictionary.size();
+ int2Bytes(dictSize, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ long[] dictArray = new long[dictSize];
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictionary.get(i);
+ }
+
+
+ encode_pos = bitPacking(dictArray, 64, encode_pos, encoded_result, dictSize);
+
+ int rle_index = 0;
+ int[] run_length = new int[remainder];
+ int[] rle_values = new int[remainder];
+
+ int previous = encodedIndices[0];
+
+ for (int j = 1; j < remainder; j++) {
+ if (encodedIndices[j] != previous) {
+ run_length[rle_index] = j;
+ rle_values[rle_index] = previous;
+ rle_index++;
+ previous = encodedIndices[j];
+ }
+ }
+
+ run_length[rle_index] = remainder;
+ rle_values[rle_index] = previous;
+ rle_index++;
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int2Bytes(rle_index, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int bw = bitWidth(remainder);
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, rle_index);
+
+ encode_pos = bitPacking(rle_values, indexBitWidth, encode_pos, encoded_result, rle_index);
+
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ int dictSize = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long[] dictionary = new long[dictSize];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 64, dictSize, dictionary);
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int[] encodedIndices = new int[remainder];
+
+ int rle_index = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int bw = bitWidth(remainder);
+
+ int[] run_length = new int[rle_index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, rle_index, run_length);
+
+ int[] rle_values = new int[rle_index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth, rle_index, rle_values);
+
+ int pos = 0;
+ for (int i = 0; i < rle_index; i++) {
+ int length = run_length[i] - pos;
+ int value = rle_values[i];
+ for (int j = 0; j < length; j++) {
+ encodedIndices[pos] = value;
+ pos++;
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = dictionary[encodedIndices[i]];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "dictionary_long.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 17];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Dictionary",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryMaterializeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryMaterializeTest.java
new file mode 100644
index 0000000..3c38562
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionaryMaterializeTest.java
@@ -0,0 +1,1017 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import javax.management.Query;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class DictionaryMaterializeTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ Map<Long, Integer> valueToIndex,
+ List<Long> dictionary) {
+ int[] encodedIndices = new int[remaining];
+
+ int dictIndex = 0;
+
+ for (int j = 0; j < remaining; j++) {
+ long value = ts_block[i * block_size + j];
+
+ if (!valueToIndex.containsKey(value)) {
+ valueToIndex.put(value, dictIndex);
+ dictionary.add(value);
+ encodedIndices[j] = dictIndex;
+ dictIndex++;
+ } else {
+ encodedIndices[j] = valueToIndex.get(value);
+ }
+ }
+
+ return encodedIndices;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+ Map<Long, Integer> valueToIndex = new LinkedHashMap<>();
+ List<Long> dictionary = new ArrayList<>();
+
+ long[] min_delta = new long[3];
+
+ int[] encodedIndices = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta, valueToIndex, dictionary);
+
+ int dictSize = dictionary.size();
+ int2Bytes(dictSize, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ long[] dictArray = new long[dictSize];
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictionary.get(i);
+ }
+
+ encode_pos = bitPacking(dictArray, 64, encode_pos, encoded_result, dictSize);
+
+ // long min_dict_value = Long.MAX_VALUE;
+ // long max_dict_value = Long.MIN_VALUE;
+
+ // for (int i = 0; i < dictSize; i++) {
+ // dictArray[i] = dictionary.get(i);
+ // if (dictArray[i] < min_dict_value) {
+ // min_dict_value = dictArray[i];
+ // }
+ // if (dictArray[i] > max_dict_value) {
+ // max_dict_value = dictArray[i];
+ // }
+ // }
+
+ // for (int i = 0; i < dictSize; i++) {
+ // dictArray[i] = dictArray[i] - min_dict_value;
+ // }
+
+ // long2Bytes(min_dict_value, encode_pos, encoded_result);
+ // encode_pos += 8;
+
+ // int bw = bitWidth(max_dict_value - min_dict_value);
+ // int2Bytes(bw, encode_pos, encoded_result);
+ // encode_pos += 4;
+
+ // encode_pos = bitPacking(dictArray, bw, encode_pos, encoded_result, dictSize);
+
+ // int rle_index = 0;
+ // int[] run_length = new int[remainder];
+ // int[] rle_values = new int[remainder];
+
+ // int previous = encodedIndices[0];
+
+ // for (int j = 1; j < remainder; j++) {
+ // if (encodedIndices[j] != previous) {
+ // run_length[rle_index] = j;
+ // rle_values[rle_index] = previous;
+ // rle_index++;
+ // previous = encodedIndices[j];
+ // }
+ // }
+
+ // run_length[rle_index] = remainder;
+ // rle_values[rle_index] = previous;
+ // rle_index++;
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ // int2Bytes(rle_index, encode_pos, encoded_result);
+ // encode_pos += 4;
+
+ // int bw = bitWidth(remainder);
+
+ // encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result,
+ // rle_index);
+
+ // encode_pos = bitPacking(rle_values, indexBitWidth, encode_pos,
+ // encoded_result, rle_index);
+
+ encode_pos = bitPacking(encodedIndices, indexBitWidth, encode_pos, encoded_result, remainder);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ int dictSize = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ // long min_dict_value = bytes2Long(encoded_result, encode_pos, 8);
+ // encode_pos += 8;
+
+ // int dictBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ // encode_pos += 4;
+
+ long[] dictionary = new long[dictSize];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dictBitWidth,
+ // dictSize, dictionary);
+
+ // for (int i = 0; i < dictSize; i++) {
+ // dictionary[i] = dictionary[i] + min_dict_value;
+ // }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 64, dictSize,
+ dictionary);
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int[] encodedIndices = new int[remainder];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth, remainder, encodedIndices);
+
+ // int rle_index = bytes2Integer(encoded_result, encode_pos, 4);
+ // encode_pos += 4;
+
+ // int bw = bitWidth(remainder);
+
+ // int[] run_length = new int[rle_index];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, rle_index,
+ // run_length);
+
+ // int[] rle_values = new int[rle_index];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth,
+ // rle_index, rle_values);
+
+ // int pos = 0;
+ // for (int i = 0; i < rle_index; i++) {
+ // int length = run_length[i] - pos;
+ // int value = rle_values[i];
+ // for (int j = 0; j < length; j++) {
+ // encodedIndices[pos] = value;
+ // pos++;
+ // }
+ // }
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = dictionary[encodedIndices[i]];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static void Query(byte[] encoded_result, long upper_bound, int[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ if (value < upper_bound) {
+ result[result_length[0]] = num_blocks * block_size + i;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ int dictSize = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ // long min_dict_value = bytes2Long(encoded_result, encode_pos, 8);
+ // encode_pos += 8;
+
+ // int dictBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ // encode_pos += 4;
+
+ long[] dictionary = new long[dictSize];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dictBitWidth,
+ // dictSize, dictionary);
+
+ // for (int i = 0; i < dictSize; i++) {
+ // dictionary[i] = dictionary[i] + min_dict_value;
+ // }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 64, dictSize,
+ dictionary);
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int[] encodedIndices = new int[remainder];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth, remainder, encodedIndices);
+
+ int start_index = block_index * block_size;
+
+ for (int i = 0; i < remainder; i++) {
+ if (dictionary[encodedIndices[i]] < upper_bound) {
+ result[result_length[0]] = start_index + i;
+ result_length[0]++;
+ }
+ }
+
+ return encode_pos;
+
+ }
+
+ public static double computeSelectivity(long len1_0, long len2_0, long halfSize, long match) {
+ double sA = Double.NaN, sB = Double.NaN, pAB = Double.NaN, lift = Double.NaN;
+
+ if (halfSize <= 0)
+ return lift;
+
+ // 基本概率
+ sA = (double) len1_0 / (double) halfSize;
+ sB = (double) len2_0 / (double) halfSize;
+ pAB = (double) match / (double) halfSize;
+
+ // lift = P(A∧B) / (P(A) P(B))，仅在分母非零时计算
+ if (sA > 0.0 && sB > 0.0) {
+ lift = pAB / (sA * sB);
+ }
+
+ return lift;
+ }
+
+ public static double phiCoefficient(long len1_0, long len2_0, long halfSize, long match) {
+ // 2x2 表格元素
+ double a = (double) match; // A ∧ B
+ double b = (double) (len1_0 - match); // A ∧ ¬B
+ double c = (double) (len2_0 - match); // ¬A ∧ B
+ double d = (double) (halfSize - (match + (len1_0 - match) + (len2_0 - match)));
+ // 等价于: d = halfSize - (a + b + c)
+
+ // 如果任何分量为负，输入可能不合法，返回 NaN
+ if (a < 0 || b < 0 || c < 0 || d < 0) {
+ return Double.NaN;
+ }
+
+ double numerator = a * d - b * c;
+ double denomTerm1 = (a + b) * (c + d);
+ double denomTerm2 = (a + c) * (b + d);
+
+ // 分母为 sqrt( denomTerm1 * denomTerm2 )
+ double denomProduct = denomTerm1 * denomTerm2;
+ if (denomProduct <= 0.0) {
+ return Double.NaN; // 避免除零或根号负数
+ }
+
+ double phi = numerator / Math.sqrt(denomProduct);
+ return phi;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = "D:/encoding-subcolumn/result/materialization/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "cstore_materialization1.csv";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 75000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ // "Encoding Time",
+ "Decoding Time",
+ "Selectivity",
+ "Phi",
+ "Points",
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++)
+ col1_data[i] = (long) (data1.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++)
+ col2_data[i] = (long) (data1.get(i + halfSize) * max_mul);
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 13];
+ byte[] encoded_result2 = new byte[col2_data.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ long tStart = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(col1_data, block_size, encoded_result1);
+ length2 = Encoder(col2_data, block_size, encoded_result2);
+ }
+
+ long tEnd = System.nanoTime();
+ encodeTime += ((tEnd - tStart) / repeatTime);
+ compressed_size += length1 + length2;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ int upper = queryRange.get(datasetName);
+
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ double selectivity = 0;
+ double phi = 0;
+ int match = 0;
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ // CompletableFuture<Void> future1 = CompletableFuture.runAsync(() -> {
+ // Query(encoded_result1, upper, res1, len1);
+ // });
+
+ // CompletableFuture<Void> future2 = CompletableFuture.runAsync(() -> {
+ // Query(encoded_result2, upper, res2, len2);
+ // });
+
+ // // 等待两个查询都完成
+ // try {
+ // CompletableFuture.allOf(future1, future2).get();
+ // } catch (InterruptedException | ExecutionException e) {
+ // e.printStackTrace();
+ // // 处理异常，可能需要中断循环或采取其他措施
+ // Thread.currentThread().interrupt(); // 重新设置中断状态
+ // break;
+ // }
+
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ long[] bits1 = new long[(halfSize + 63) / 64];
+ long[] bits2 = new long[(halfSize + 63) / 64];
+
+ // 设置bit
+ for (int i = 0; i < len1[0]; i++) {
+ int pos = res1[i];
+ bits1[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ for (int i = 0; i < len2[0]; i++) {
+ int pos = res2[i];
+ bits2[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ // 求交集并计数
+ match = 0;
+ for (int i = 0; i < bits1.length; i++) {
+ long intersection = bits1[i] & bits2[i];
+ match += Long.bitCount(intersection);
+ }
+ }
+ tEnd = System.nanoTime();
+
+ selectivity = computeSelectivity(len1[0], len2[0], halfSize, match);
+ phi = phiCoefficient(len1[0], len2[0], halfSize, match);
+ System.out.println(len1[0] + "," + len2[0]);
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+ // long[] data2_arr_decoded = new long[data2_arr.length];
+
+ // s = System.nanoTime();
+
+ // int[] result = new int[data2_arr.length];
+ // int[] result_length = new int[1];
+
+ // for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // // data2_arr_decoded = Decoder(encoded_result);
+ // Query(encoded_result, queryRange.get(datasetName), result, result_length);
+ // }
+
+ String[] record = {
+ datasetName,
+ "Dictionary",
+ // String.valueOf(encodeTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(selectivity),
+ String.valueOf(phi),
+ String.valueOf(totalSize)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionarySubcolumnLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionarySubcolumnLongTest.java
new file mode 100644
index 0000000..7b0d83b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/DictionarySubcolumnLongTest.java
@@ -0,0 +1,749 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class DictionarySubcolumnLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ Map<Long, Integer> valueToIndex,
+ List<Long> dictionary) {
+ int[] encodedIndices = new int[remaining];
+
+ int dictIndex = 0;
+
+ for (int j = 0; j < remaining; j++) {
+ long value = ts_block[i * block_size + j];
+
+ if (!valueToIndex.containsKey(value)) {
+ valueToIndex.put(value, dictIndex);
+ dictionary.add(value);
+ encodedIndices[j] = dictIndex;
+ dictIndex++;
+ } else {
+ encodedIndices[j] = valueToIndex.get(value);
+ }
+ }
+
+ return encodedIndices;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ Map<Long, Integer> valueToIndex = new LinkedHashMap<>();
+ List<Long> dictionary = new ArrayList<>();
+
+ long[] min_delta = new long[3];
+
+ int[] encodedIndices = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta, valueToIndex, dictionary);
+
+ int dictSize = dictionary.size();
+ int2Bytes(dictSize, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ long[] dictArray = new long[dictSize];
+
+ long min_dict_value = Long.MAX_VALUE;
+ long max_dict_value = Long.MIN_VALUE;
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictionary.get(i);
+ if (dictArray[i] < min_dict_value) {
+ min_dict_value = dictArray[i];
+ }
+ if (dictArray[i] > max_dict_value) {
+ max_dict_value = dictArray[i];
+ }
+ }
+
+ for (int i = 0; i < dictSize; i++) {
+ dictArray[i] = dictArray[i] - min_dict_value;
+ }
+
+ long2Bytes(min_dict_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_dict_value - min_dict_value);
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // encode_pos = bitPacking(dictArray, bw, encode_pos, encoded_result, dictSize);
+
+ encode_pos = SubcolumnLongTest.SubcolumnEncoder(dictArray, encode_pos, encoded_result, beta, dictSize);
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ // encode_pos = bitPacking(encodedIndices, indexBitWidth, encode_pos, encoded_result, remainder);
+
+ encode_pos = SubcolumnTest.SubcolumnEncoder(encodedIndices, encode_pos, encoded_result, beta, remainder);
+
+ // encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ // encoded_result, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ int dictSize = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long min_dict_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int dictBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ long[] dictionary = new long[dictSize];
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dictBitWidth, dictSize, dictionary);
+
+ encode_pos = SubcolumnLongTest.SubcolumnDecoder(encoded_result, encode_pos, dictionary, dictSize);
+
+ for (int i = 0; i < dictSize; i++) {
+ dictionary[i] = dictionary[i] + min_dict_value;
+ }
+
+ int maxIndex = dictSize - 1;
+ int indexBitWidth = maxIndex > 0 ? bitWidth(maxIndex) : 1;
+
+ int[] encodedIndices = new int[remainder];
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, indexBitWidth, remainder, encodedIndices);
+
+ encode_pos = SubcolumnTest.SubcolumnDecoder(encoded_result, encode_pos, encodedIndices, remainder);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = dictionary[encodedIndices[i]];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 3;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "dictionary_subcolumn_long.csv";
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 17];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPFORCompressionBenchmarkTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPFORCompressionBenchmarkTest.java
new file mode 100644
index 0000000..93926b0
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPFORCompressionBenchmarkTest.java
@@ -0,0 +1,228 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import me.lemire.integercompression.FastPFOR;
+import me.lemire.integercompression.IntCompressor;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Benchmarks {@link FastPFOR} (JavaFastPFOR, same family as FastPFOR C++ {@code fastpfor.h})
+ * compression ratio and encode/decode time, following the CSV workflow in
+ * {@link SubcolumnAddDictionaryTest#test0()}.
+ *
+ * <p>FastPFOR requires the input length to be a multiple of {@link FastPFOR#BLOCK_SIZE} (256).
+ * Inputs are zero-padded for compression; ratios use the original point count in the denominator.
+ */
+public class FastPFORCompressionBenchmarkTest {
+
+ private static final int REPEAT_WARMUP = 5;
+ private static final int REPEAT_TIMED = 200;
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ static int paddedLength(int length, int blockSize) {
+ int r = length % blockSize;
+ return r == 0 ? length : length + (blockSize - r);
+ }
+
+ static int[] padForFastPFor(int[] data) {
+ int m = paddedLength(data.length, FastPFOR.BLOCK_SIZE);
+ if (m == data.length) {
+ return data;
+ }
+ return Arrays.copyOf(data, m);
+ }
+
+ @Test
+ public void testRoundTripPadded() {
+ Random rnd = new Random(7);
+ for (int n : new int[] {256, 512, 1024, 4096}) {
+ int[] a = new int[n];
+ for (int i = 0; i < n; i++) {
+ a[i] = rnd.nextInt(1 << 18);
+ }
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ int[] c = comp.compress(a);
+ int[] b = comp.uncompress(c);
+ assertEquals(a.length, b.length);
+ for (int i = 0; i < a.length; i++) {
+ assertEquals(a[i], b[i]);
+ }
+ }
+ }
+
+ @Test
+ public void testRoundTripWithPadding() {
+ int[] shortData = new int[100];
+ for (int i = 0; i < shortData.length; i++) {
+ shortData[i] = i * 3;
+ }
+ int[] padded = padForFastPFor(shortData);
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ int[] c = comp.compress(padded);
+ int[] b = comp.uncompress(c);
+ for (int i = 0; i < shortData.length; i++) {
+ assertEquals(shortData[i], b[i]);
+ }
+ }
+
+ @Test
+ public void testSyntheticCompressionStats() {
+ int n = 50_000;
+ int nPad = paddedLength(n, FastPFOR.BLOCK_SIZE);
+ int[] data = new int[nPad];
+ for (int i = 0; i < n; i++) {
+ data[i] = i % 17;
+ }
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ for (int r = 0; r < REPEAT_WARMUP; r++) {
+ comp.uncompress(comp.compress(data));
+ }
+ int[] compressed = comp.compress(data);
+ long t0 = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ compressed = comp.compress(data);
+ }
+ long encNs = (System.nanoTime() - t0) / REPEAT_TIMED;
+ int[] decoded = null;
+ t0 = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ decoded = comp.uncompress(compressed);
+ }
+ long decNs = (System.nanoTime() - t0) / REPEAT_TIMED;
+ for (int i = 0; i < n; i++) {
+ assertEquals(data[i], decoded[i]);
+ }
+ double rawBytes = (double) n * Integer.BYTES;
+ double compBytes = (double) compressed.length * Integer.BYTES;
+ double ratio = compBytes / rawBytes;
+ System.out.printf(
+ "FastPFOR synthetic: n=%d (padded to %d) encode=%d ns decode=%d ns compressed=%d B ratio=%.4f (vs original n)%n",
+ n, nPad, encNs, decNs, (int) compBytes, ratio);
+ org.junit.Assert.assertTrue("expect reasonable size on low-entropy data", ratio < 0.5);
+ }
+
+ @Test
+ public void testCsvDatasetsIfPresent() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ File directory = new File(inputParentDir);
+ Assume.assumeTrue(
+ "Skip CSV benchmark when " + inputParentDir + " is missing", directory.isDirectory());
+
+ String outputParentDir = parentDir + "result/";
+ new File(outputParentDir).mkdirs();
+ String outputPath = outputParentDir + "fastpfor_compression.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Algorithm",
+ "Encode ns",
+ "Decode ns",
+ "Points",
+ "Padded to",
+ "Compressed bytes",
+ "Ratio (compressed / raw int32 for original points)"
+ });
+
+ IntCompressor comp = new IntCompressor(new FastPFOR());
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ Assume.assumeTrue(csvFiles != null);
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> floats = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, getDecimalPrecision(fStr));
+ floats.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ int[] raw = new int[floats.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < floats.size(); i++) {
+ raw[i] = (int) (floats.get(i) * maxMul);
+ }
+ int[] padded = padForFastPFor(raw);
+
+ for (int r = 0; r < REPEAT_WARMUP; r++) {
+ comp.uncompress(comp.compress(padded));
+ }
+ int[] compressed = comp.compress(padded);
+ long s = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ compressed = comp.compress(padded);
+ }
+ long encNs = (System.nanoTime() - s) / REPEAT_TIMED;
+
+ int[] decoded = comp.uncompress(compressed);
+ s = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ decoded = comp.uncompress(compressed);
+ }
+ long decNs = (System.nanoTime() - s) / REPEAT_TIMED;
+
+ for (int i = 0; i < raw.length; i++) {
+ assertEquals(raw[i], decoded[i]);
+ }
+ int compBytes = compressed.length * Integer.BYTES;
+ double ratio = compBytes / (double) (raw.length * (long) Integer.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "FastPFOR",
+ Long.toString(encNs),
+ Long.toString(decNs),
+ Integer.toString(raw.length),
+ Integer.toString(padded.length),
+ Integer.toString(compBytes),
+ Double.toString(ratio)
+ });
+ System.out.printf("%s FastPFOR ratio=%.4f (padded %d -> %d)%n", datasetName, ratio, raw.length, padded.length);
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPForSimple8bCodec.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPForSimple8bCodec.java
new file mode 100644
index 0000000..75e021f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/FastPForSimple8bCodec.java
@@ -0,0 +1,440 @@
+package org.apache.iotdb.tsfile.encoding;
+
+/**
+ * Java port of Simple8b<MarkLength=true> from FastPFOR ({@code headers/simple8b.h}). Reference:
+ * Vo Ngoc Anh, Alistair Moffat, "Index compression using 64-bit words", Software: Practice and
+ * Experience 40(2), 2010; Daniel Lemire's FastPFOR library.
+ *
+ * <p>Compressed layout matches the C++ codec: first uint32 is original length, followed by 64-bit
+ * words stored as little-endian uint32 pairs.
+ */
+final class FastPForSimple8bCodec {
+
+ private static final int SIMPLE8B_LOGDESC = 4;
+
+ private FastPForSimple8bCodec() {}
+
+ static int which(long w) {
+ return (int) (w >>> (64 - SIMPLE8B_LOGDESC));
+ }
+
+ private static boolean tryMe(int[] in, int ip, int remaining, int num1, int log1) {
+ int n = Math.min(remaining, num1);
+ if (log1 >= 32) {
+ return true;
+ }
+ long limit = 1L << log1;
+ for (int i = 0; i < n; i++) {
+ if ((in[ip + i] & 0xffffffffL) >= limit) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean tryMeFull(int[] in, int ip, int num1, int log1) {
+ if (log1 >= 32) {
+ return true;
+ }
+ long limit = 1L << log1;
+ for (int i = 0; i < num1; i++) {
+ if ((in[ip + i] & 0xffffffffL) >= limit) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static long maskFor(int log1) {
+ if (log1 >= 32) {
+ return 0xffffffffL;
+ }
+ return (1L << log1) - 1;
+ }
+
+ private static long encodeWord(int[] in, int ip, int valuesRemaining, boolean useFull240Path) {
+ long w;
+ if (useFull240Path && valuesRemaining >= 240) {
+ if (tryMeFull(in, ip, 120, 0) && tryMeFull(in, ip + 120, 120, 0)) {
+ return 0L;
+ }
+ if (tryMeFull(in, ip, 120, 0)) {
+ return 1L << (64 - SIMPLE8B_LOGDESC);
+ }
+ }
+ if (tryMe(in, ip, valuesRemaining, 60, 1)) {
+ w = 2;
+ int coded = Math.min(valuesRemaining, 60);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 1) | (in[ip + i] & 1L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 30, 2)) {
+ w = 3;
+ int coded = Math.min(valuesRemaining, 30);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 2) | (in[ip + i] & 3L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 2 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 20, 3)) {
+ w = 4;
+ int coded = Math.min(valuesRemaining, 20);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 3) | (in[ip + i] & 7L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 3 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 15, 4)) {
+ w = 5;
+ int coded = Math.min(valuesRemaining, 15);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 4) | (in[ip + i] & 15L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 4 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 12, 5)) {
+ w = 6;
+ int coded = Math.min(valuesRemaining, 12);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 5) | (in[ip + i] & 31L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 5 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 10, 6)) {
+ w = 7;
+ int coded = Math.min(valuesRemaining, 10);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 6) | (in[ip + i] & 63L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 6 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 8, 7)) {
+ w = 8;
+ int coded = Math.min(valuesRemaining, 8);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 7) | (in[ip + i] & 127L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 7 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 7, 8)) {
+ w = 9;
+ int coded = Math.min(valuesRemaining, 7);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 8) | (in[ip + i] & 255L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 8 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 6, 10)) {
+ w = 10;
+ int coded = Math.min(valuesRemaining, 6);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 10) | (in[ip + i] & 1023L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 10 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 5, 12)) {
+ w = 11;
+ int coded = Math.min(valuesRemaining, 5);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 12) | (in[ip + i] & 4095L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 12 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 4, 15)) {
+ w = 12;
+ int coded = Math.min(valuesRemaining, 4);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 15) | (in[ip + i] & 32767L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 15 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 3, 20)) {
+ w = 13;
+ int coded = Math.min(valuesRemaining, 3);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 20) | (in[ip + i] & 1048575L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 20 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 2, 30)) {
+ w = 14;
+ int coded = Math.min(valuesRemaining, 2);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 30) | (in[ip + i] & 1073741823L);
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 30 * coded);
+ return w;
+ }
+ if (tryMe(in, ip, valuesRemaining, 1, 60)) {
+ w = 15;
+ int coded = Math.min(valuesRemaining, 1);
+ for (int i = 0; i < coded; i++) {
+ w = (w << 60) | (in[ip + i] & ((1L << 60) - 1));
+ }
+ w <<= (64 - SIMPLE8B_LOGDESC - 60 * coded);
+ return w;
+ }
+ throw new IllegalStateException("Simple8b: no case applies");
+ }
+
+ /** @return number of input values consumed for this word */
+ private static int codedCountForWord(long word, int valuesRemaining) {
+ switch (which(word)) {
+ case 0:
+ return Math.min(valuesRemaining, 240);
+ case 1:
+ return Math.min(valuesRemaining, 120);
+ case 2:
+ return Math.min(valuesRemaining, 60);
+ case 3:
+ return Math.min(valuesRemaining, 30);
+ case 4:
+ return Math.min(valuesRemaining, 20);
+ case 5:
+ return Math.min(valuesRemaining, 15);
+ case 6:
+ return Math.min(valuesRemaining, 12);
+ case 7:
+ return Math.min(valuesRemaining, 10);
+ case 8:
+ return Math.min(valuesRemaining, 8);
+ case 9:
+ return Math.min(valuesRemaining, 7);
+ case 10:
+ return Math.min(valuesRemaining, 6);
+ case 11:
+ return Math.min(valuesRemaining, 5);
+ case 12:
+ return Math.min(valuesRemaining, 4);
+ case 13:
+ return Math.min(valuesRemaining, 3);
+ case 14:
+ return Math.min(valuesRemaining, 2);
+ case 15:
+ return Math.min(valuesRemaining, 1);
+ default:
+ throw new IllegalStateException("bad selector");
+ }
+ }
+
+ /** Encode with length prefix (MarkLength=true). Output: [length][u64 as LE pair]... */
+ static int[] encode(int[] in) {
+ int length = in.length;
+ java.util.ArrayList<Long> words = new java.util.ArrayList<>(length / 60 + 8);
+ int ip = 0;
+ int valuesRemaining = length;
+ while (valuesRemaining >= 240) {
+ long word = encodeWord(in, ip, valuesRemaining, true);
+ int coded = codedCountForWord(word, valuesRemaining);
+ words.add(word);
+ ip += coded;
+ valuesRemaining -= coded;
+ }
+ while (valuesRemaining > 0) {
+ long word = encodeWord(in, ip, valuesRemaining, false);
+ int coded = codedCountForWord(word, valuesRemaining);
+ words.add(word);
+ ip += coded;
+ valuesRemaining -= coded;
+ }
+ int[] out = new int[1 + 2 * words.size()];
+ out[0] = length;
+ int o = 1;
+ for (long lw : words) {
+ out[o++] = (int) (lw & 0xffffffffL);
+ out[o++] = (int) (lw >>> 32);
+ }
+ return out;
+ }
+
+ private static void unpackFull(long w, int[] out, int op, int num1, int log1) {
+ long mask = maskFor(log1);
+ for (int k = 0; k < num1; k++) {
+ int sh = 64 - SIMPLE8B_LOGDESC - log1 - k * log1;
+ out[op + k] = (int) ((w >>> sh) & mask);
+ }
+ }
+
+ private static void unpackCareful(long w, int[] out, int op, int num1, int log1) {
+ long mask = maskFor(log1);
+ for (int k = 0; k < num1; k++) {
+ int sh = 64 - SIMPLE8B_LOGDESC - log1 - k * log1;
+ out[op + k] = (int) ((w >>> sh) & mask);
+ }
+ }
+
+ static int[] decode(int[] compressed) {
+ int marked = compressed[0];
+ int[] out = new int[marked];
+ int wp = 0;
+ int idx = 1;
+ while (wp < marked) {
+ long w = (compressed[idx] & 0xffffffffL) | ((long) compressed[idx + 1] << 32);
+ idx += 2;
+ int sel = which(w);
+ int left = marked - wp;
+ switch (sel) {
+ case 0:
+ if (left > 240) {
+ unpackFull(w, out, wp, 240, 0);
+ wp += 240;
+ } else {
+ unpackCareful(w, out, wp, left, 0);
+ wp += left;
+ }
+ break;
+ case 1:
+ if (left > 120) {
+ unpackFull(w, out, wp, 120, 0);
+ wp += 120;
+ } else {
+ unpackCareful(w, out, wp, left, 0);
+ wp += left;
+ }
+ break;
+ case 2:
+ if (left > 60) {
+ unpackFull(w, out, wp, 60, 1);
+ wp += 60;
+ } else {
+ unpackCareful(w, out, wp, left, 1);
+ wp += left;
+ }
+ break;
+ case 3:
+ if (left > 30) {
+ unpackFull(w, out, wp, 30, 2);
+ wp += 30;
+ } else {
+ unpackCareful(w, out, wp, left, 2);
+ wp += left;
+ }
+ break;
+ case 4:
+ if (left > 20) {
+ unpackFull(w, out, wp, 20, 3);
+ wp += 20;
+ } else {
+ unpackCareful(w, out, wp, left, 3);
+ wp += left;
+ }
+ break;
+ case 5:
+ if (left > 15) {
+ unpackFull(w, out, wp, 15, 4);
+ wp += 15;
+ } else {
+ unpackCareful(w, out, wp, left, 4);
+ wp += left;
+ }
+ break;
+ case 6:
+ if (left > 12) {
+ unpackFull(w, out, wp, 12, 5);
+ wp += 12;
+ } else {
+ unpackCareful(w, out, wp, left, 5);
+ wp += left;
+ }
+ break;
+ case 7:
+ if (left > 10) {
+ unpackFull(w, out, wp, 10, 6);
+ wp += 10;
+ } else {
+ unpackCareful(w, out, wp, left, 6);
+ wp += left;
+ }
+ break;
+ case 8:
+ if (left > 8) {
+ unpackFull(w, out, wp, 8, 7);
+ wp += 8;
+ } else {
+ unpackCareful(w, out, wp, left, 7);
+ wp += left;
+ }
+ break;
+ case 9:
+ if (left > 7) {
+ unpackFull(w, out, wp, 7, 8);
+ wp += 7;
+ } else {
+ unpackCareful(w, out, wp, left, 8);
+ wp += left;
+ }
+ break;
+ case 10:
+ if (left > 6) {
+ unpackFull(w, out, wp, 6, 10);
+ wp += 6;
+ } else {
+ unpackCareful(w, out, wp, left, 10);
+ wp += left;
+ }
+ break;
+ case 11:
+ if (left > 5) {
+ unpackFull(w, out, wp, 5, 12);
+ wp += 5;
+ } else {
+ unpackCareful(w, out, wp, left, 12);
+ wp += left;
+ }
+ break;
+ case 12:
+ if (left > 4) {
+ unpackFull(w, out, wp, 4, 15);
+ wp += 4;
+ } else {
+ unpackCareful(w, out, wp, left, 15);
+ wp += left;
+ }
+ break;
+ case 13:
+ if (left > 3) {
+ unpackFull(w, out, wp, 3, 20);
+ wp += 3;
+ } else {
+ unpackCareful(w, out, wp, left, 20);
+ wp += left;
+ }
+ break;
+ case 14:
+ if (left > 2) {
+ unpackFull(w, out, wp, 2, 30);
+ wp += 2;
+ } else {
+ unpackCareful(w, out, wp, left, 30);
+ wp += left;
+ }
+ break;
+ case 15:
+ unpackCareful(w, out, wp, 1, 60);
+ wp += 1;
+ break;
+ default:
+ throw new IllegalStateException("Simple8b decode: bad selector");
+ }
+ }
+ return out;
+ }
+
+ static int compressedSizeBytes(int[] compressed) {
+ return compressed.length * Integer.BYTES;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndex.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndex.java
new file mode 100644
index 0000000..4e07b46
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndex.java
@@ -0,0 +1,261 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.*;
+
+public class HBPIndex {
+
+ /** width of processor word */
+ public static final int W = 64;
+ // public static final int W = 8;
+
+ /** number of bits per code (k) */
+ public final int k;
+
+ /** section width = k + 1 (delimiter bit + k code bits) */
+ public final int sectionBits;
+
+ /** how many sections per 64-bit word */
+ public final int sectionsPerWord;
+
+ /** number of codes per segment = (k+1) * sectionsPerWord (≤ W) */
+ public final int codesPerSegment;
+
+ /** total number of codes (rows) */
+ public final int n;
+
+ /** number of segments */
+ public final int segments;
+
+ /**
+ * Packed storage: for each segment s (0..segments-1), we store (k+1) words
+ * v[0..k].
+ * Layout: words[(s * (k+1)) + i] corresponds to vi (i in 0..k).
+ */
+ public final long[] words;
+
+ /** Masks repeated per section */
+ public final long lowKOnesRepeat; // 01^k01^k...01^k mask
+ public final long delimiterBitRepeat; // 10^k10^k...10^k mask
+ public final long addOneEachSection; // ...0001 in each section (LSB 1)
+
+ public enum Op {
+ EQ, NE, LT, LE, GT, GE
+ }
+
+ /**
+ * Build HBP storage from k-bit codes.
+ *
+ * @param kBits number of bits per code (1..63)
+ * @param codes encoded values in [0, 2^kBits)
+ */
+ public HBPIndex(int kBits, int[] codes) {
+ // if (kBits <= 0 || kBits >= W) {
+ // throw new IllegalArgumentException("k must be in [1, 63]");
+ // }
+
+ this.k = kBits;
+ this.sectionBits = k + 1;
+ this.sectionsPerWord = W / sectionBits; // floor
+ if (sectionsPerWord <= 0)
+ throw new IllegalArgumentException("k too large for 64-bit word");
+ this.codesPerSegment = sectionsPerWord * (k + 1);
+ this.n = codes.length;
+ this.segments = (n + codesPerSegment - 1) / codesPerSegment;
+ this.words = new long[segments * (k + 1)];
+
+ // build masks
+ this.lowKOnesRepeat = repeatInSections((1L << k) - 1L); // 01^k
+ this.delimiterBitRepeat = repeatInSections(1L << k); // 10^k
+ this.addOneEachSection = repeatInSections(1L); // ...0001
+
+ // pack codes
+ pack(codes);
+ }
+
+ /* ---------- Public API ---------- */
+
+ /** Return a BitSet with one bit per row; bit=1 means row selected by op C. */
+ public BitSet select(Op op, int C) {
+ return selectInternal(op, C & ((1 << k) - 1));
+ }
+
+ /** Count matches for op C. */
+ public long count(Op op, int C) {
+ BitSet bs = select(op, C);
+ return bs.cardinality();
+ }
+
+ /** Total rows */
+ public int size() {
+ return n;
+ }
+
+ /** Debug: fetch original k-bit code back (slow path). */
+ public int getCode(int index) {
+ if (index < 0 || index >= n)
+ throw new IndexOutOfBoundsException();
+ int s = index / codesPerSegment;
+ int posInSeg = index - s * codesPerSegment; // 0..(codesPerSegment-1)
+ // Which word vi holds this code in the segment?
+ int i = posInSeg % (k + 1); // word index (v_i)
+ int j = posInSeg / (k + 1); // section index inside that word
+ long word = words[s * (k + 1) + i];
+ long section = (word >>> (j * sectionBits)) & ((1L << sectionBits) - 1L);
+ // section: [delimiter bit (MSB)=0][k code bits]
+ return (int) (section & ((1L << k) - 1L));
+ }
+
+ /* ---------- Core HBP packing & scan ---------- */
+
+ public void pack(int[] codes) {
+ long sectionMask = (1L << sectionBits) - 1L;
+ for (int s = 0; s < segments; s++) {
+ int base = s * codesPerSegment;
+ int upto = Math.min(n, base + codesPerSegment);
+ int count = upto - base;
+ // prepare k+1 words for this segment
+ for (int i = 0; i <= k; i++) {
+ long w = 0L;
+ for (int j = 0; j < sectionsPerWord; j++) {
+ int idx = base + i + j * (k + 1);
+ if (idx >= upto)
+ break;
+ int code = codes[idx] & ((1 << k) - 1);
+ long section = (long) code; // delimiter (MSB) left as 0
+ w |= (section & sectionMask) << (j * sectionBits);
+ }
+ words[s * (k + 1) + i] = w;
+ }
+ }
+ }
+
+ public BitSet selectInternal(Op op, int Ck) {
+ BitSet out = new BitSet(n);
+ long yRepeat = repeatInSections(Ck & ((1 << k) - 1));
+ for (int s = 0; s < segments; s++) {
+ long segBits = 0L;
+ // compute ms := OR_{i=0..k} ( f◦(v_i, C) >>> i )
+ for (int i = 0; i <= k; i++) {
+ // System.out.println("current segment = " + s + ", word index = " + i);
+
+ long X = words[s * (k + 1) + i];
+ long Z = fOp(op, X, yRepeat);
+
+ // System.out.println("Z = " + String.format("%64s", Long.toBinaryString(Z)).replace(' ', '0'));
+
+ // long shifted = (i == 0) ? Z : (Z >>> i);
+ long shifted = Z >>> (k - i);
+ segBits |= shifted;
+ }
+
+ // System.out.println("segBits = " + String.format("%64s", Long.toBinaryString(segBits)).replace(' ', '0'));
+
+ // mask off any high bits beyond rows in last (possibly partial) segment
+ int base = s * codesPerSegment;
+ int valid = Math.min(codesPerSegment, n - base);
+ long mask = valid == 64 ? ~0L : ((1L << valid) - 1L);
+ segBits &= mask;
+
+ // System.out.println("masked segBits = " + String.format("%64s", Long.toBinaryString(segBits)).replace(' ', '0'));
+
+ // write into BitSet
+ int bitIndexBase = s * codesPerSegment;
+ while (segBits != 0) {
+ int t = Long.numberOfTrailingZeros(segBits);
+ out.set(bitIndexBase + t);
+ segBits &= (segBits - 1); // clear lowest set bit
+ }
+ }
+ return out;
+ }
+
+ /**
+ * f◦ on one 64-bit word of packed sections, returns delimiter-bit-only result
+ * (10^k in true sections).
+ */
+ public long fOp(Op op, long X, long Yrepeat) {
+ // return switch (op) {
+ // // case NE -> ((X ^ Yrepeat) + lowKOnesRepeat) & delimiterBitRepeat;
+ // // case EQ -> (~((X ^ Yrepeat) + lowKOnesRepeat)) & delimiterBitRepeat;
+ // // case LT -> (Yrepeat + (X ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ // // case LE -> (Yrepeat + (X ^ lowKOnesRepeat) + addOneEachSection) &
+ // delimiterBitRepeat;
+ // // case GT -> (X + (Yrepeat ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ // // case GE -> (X + (Yrepeat ^ lowKOnesRepeat) + addOneEachSection) &
+ // delimiterBitRepeat;
+ //
+ // };
+
+ switch (op) {
+ case NE:
+ return ((X ^ Yrepeat) + lowKOnesRepeat) & delimiterBitRepeat;
+ case EQ:
+ return (~((X ^ Yrepeat) + lowKOnesRepeat)) & delimiterBitRepeat;
+ case LT:
+ return (Yrepeat + (X ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ case LE:
+ return (Yrepeat + (X ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat;
+ case GT:
+ return (X + (Yrepeat ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ case GE:
+ return (X + (Yrepeat ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat;
+ }
+
+ return 0L;
+ }
+
+ /**
+ * Repeat a (k+1)-bit payload across all sections (payload must fit into
+ * sectionBits).
+ */
+ public long repeatInSections(long payload) {
+ long res = 0L;
+ for (int j = 0; j < sectionsPerWord; j++) {
+ res |= (payload & ((1L << sectionBits) - 1L)) << (j * sectionBits);
+ }
+ return res;
+ }
+
+ /* ---------- Demo ---------- */
+
+ public static void main(String[] args) {
+ // Running example from the paper (§3.1.1): k=3, 10 codes
+ int k = 3;
+ int[] codes = {
+ 1, 5, 6, 1, 6, 4, 0, 7, 4, 3
+ };
+ HBPIndex idx = new HBPIndex(k, codes);
+
+ for (int i = 0; i < idx.words.length; i++) {
+ System.out.printf("word[%d] = %016X\n", i, idx.words[i]);
+ }
+
+ System.out.println("segments: " + idx.segments);
+
+ System.out.println(String.format("%64s", Long.toBinaryString(idx.lowKOnesRepeat)).replace(' ', '0'));
+ System.out.println(String.format("%64s", Long.toBinaryString(idx.delimiterBitRepeat)).replace(' ', '0'));
+ System.out.println(String.format("%64s", Long.toBinaryString(idx.addOneEachSection)).replace(' ', '0'));
+
+ System.out.println("n = " + idx.size());
+
+ // c < 4
+ BitSet lt5 = idx.select(Op.LT, 4);
+ System.out.println("< 4 -> " + lt5);
+
+ // c == 4
+ BitSet eq4 = idx.select(Op.EQ, 4);
+ System.out.println("= 4 -> " + eq4);
+
+ // c >= 6
+ BitSet ge6 = idx.select(Op.GE, 6);
+ System.out.println(">= 6 -> " + ge6);
+
+ // show decoded codes back (debug)
+ for (int i = 0; i < idx.size(); i++) {
+ System.out.print(idx.getCode(i) + (i + 1 == idx.size() ? "\n" : " "));
+ }
+
+ // count example
+ System.out.println("count(<5) = " + idx.count(Op.LT, 5));
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLong.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLong.java
new file mode 100644
index 0000000..0d8d862
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLong.java
@@ -0,0 +1,194 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.*;
+
+public class HBPIndexLong {
+
+ public static final int W = 64;
+
+ public final int k;
+
+ public final int sectionBits;
+
+ public final int sectionsPerWord;
+
+ public final int codesPerSegment;
+
+ public final int n;
+
+ public final int segments;
+
+ public final long[] words;
+
+ public final long lowKOnesRepeat;
+ public final long delimiterBitRepeat;
+ public final long addOneEachSection;
+
+ public enum Op {
+ EQ, NE, LT, LE, GT, GE
+ }
+
+ public HBPIndexLong(int kBits, long[] codes) {
+
+ this.k = kBits;
+ this.sectionBits = k + 1;
+ this.sectionsPerWord = W / sectionBits;
+ if (sectionsPerWord <= 0)
+ throw new IllegalArgumentException("k too large for 64-bit word");
+ this.codesPerSegment = sectionsPerWord * (k + 1);
+ this.n = codes.length;
+ this.segments = (n + codesPerSegment - 1) / codesPerSegment;
+ this.words = new long[segments * (k + 1)];
+
+ this.lowKOnesRepeat = repeatInSections((1L << k) - 1L);
+ this.delimiterBitRepeat = repeatInSections(1L << k);
+ this.addOneEachSection = repeatInSections(1L);
+
+ pack(codes);
+ }
+
+ public BitSet select(Op op, long C) {
+ return selectInternal(op, C & ((1L << k) - 1));
+ }
+
+ public long count(Op op, long C) {
+ BitSet bs = select(op, C);
+ return bs.cardinality();
+ }
+
+ public int size() {
+ return n;
+ }
+
+ public long getCode(int index) {
+ if (index < 0 || index >= n)
+ throw new IndexOutOfBoundsException();
+ int s = index / codesPerSegment;
+ int posInSeg = index - s * codesPerSegment;
+ int i = posInSeg % (k + 1);
+ int j = posInSeg / (k + 1);
+ long word = words[s * (k + 1) + i];
+ long code = 0L;
+ for (int b = 0; b < k; b += 2) {
+ int chunkBits = Math.min(2, k - b);
+ long chunk = (word >>> (j * sectionBits + b)) & ((1L << chunkBits) - 1L);
+ code |= chunk << b;
+ }
+ return code;
+ }
+
+ public void pack(long[] codes) {
+ for (int s = 0; s < segments; s++) {
+ int base = s * codesPerSegment;
+ int upto = Math.min(n, base + codesPerSegment);
+ for (int i = 0; i <= k; i++) {
+ long w = 0L;
+ for (int j = 0; j < sectionsPerWord; j++) {
+ int idx = base + i + j * (k + 1);
+ if (idx >= upto)
+ break;
+ long code = codes[idx] & ((1L << k) - 1L);
+ for (int b = 0; b < k; b += 2) {
+ int chunkBits = Math.min(2, k - b);
+ long chunk = (code >> b) & ((1L << chunkBits) - 1L);
+ w |= chunk << (j * sectionBits + b);
+ }
+ }
+ words[s * (k + 1) + i] = w;
+ }
+ }
+ }
+
+ public BitSet selectInternal(Op op, long Ck) {
+ BitSet out = new BitSet(n);
+ long yRepeat = repeatInSections(Ck & ((1L << k) - 1));
+ for (int s = 0; s < segments; s++) {
+ long segBits = 0L;
+ for (int i = 0; i <= k; i++) {
+
+ long X = words[s * (k + 1) + i];
+ long Z = fOp(op, X, yRepeat);
+
+ long shifted = Z >>> (k - i);
+ segBits |= shifted;
+ }
+
+ int base = s * codesPerSegment;
+ int valid = Math.min(codesPerSegment, n - base);
+ long mask = valid == 64 ? ~0L : ((1L << valid) - 1L);
+ segBits &= mask;
+
+ int bitIndexBase = s * codesPerSegment;
+ while (segBits != 0) {
+ int t = Long.numberOfTrailingZeros(segBits);
+ out.set(bitIndexBase + t);
+ segBits &= (segBits - 1);
+ }
+ }
+ return out;
+ }
+
+ public long fOp(Op op, long X, long Yrepeat) {
+
+ switch (op) {
+ case NE:
+ return ((X ^ Yrepeat) + lowKOnesRepeat) & delimiterBitRepeat;
+ case EQ:
+ return (~((X ^ Yrepeat) + lowKOnesRepeat)) & delimiterBitRepeat;
+ case LT:
+ return (Yrepeat + (X ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ case LE:
+ return (Yrepeat + (X ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat;
+ case GT:
+ return (X + (Yrepeat ^ lowKOnesRepeat)) & delimiterBitRepeat;
+ case GE:
+ return (X + (Yrepeat ^ lowKOnesRepeat) + addOneEachSection) & delimiterBitRepeat;
+ }
+
+ return 0L;
+ }
+
+ public long repeatInSections(long payload) {
+ long res = 0L;
+ for (int j = 0; j < sectionsPerWord; j++) {
+ res |= (payload & ((1L << sectionBits) - 1L)) << (j * sectionBits);
+ }
+ return res;
+ }
+
+
+ public static void main(String[] args) {
+ int k = 3;
+ long[] codes = {
+ 1, 5, 6, 1, 6, 4, 0, 7, 4, 3
+ };
+ HBPIndexLong idx = new HBPIndexLong(k, codes);
+
+ for (int i = 0; i < idx.words.length; i++) {
+ System.out.printf("word[%d] = %016X\n", i, idx.words[i]);
+ }
+
+ System.out.println("segments: " + idx.segments);
+
+ System.out.println("lowKOnesRepeat: " + String.format("%64s", Long.toBinaryString(idx.lowKOnesRepeat)).replace(' ', '0'));
+ System.out.println("delimiterBitRepeat: " + String.format("%64s", Long.toBinaryString(idx.delimiterBitRepeat)).replace(' ', '0'));
+ System.out.println("addOneEachSection: " + String.format("%64s", Long.toBinaryString(idx.addOneEachSection)).replace(' ', '0'));
+
+ System.out.println("n = " + idx.size());
+
+ BitSet lt5 = idx.select(Op.LT, 4);
+ System.out.println("< 4 -> " + lt5);
+
+ BitSet eq4 = idx.select(Op.EQ, 4);
+ System.out.println("= 4 -> " + eq4);
+
+ BitSet ge6 = idx.select(Op.GE, 6);
+ System.out.println(">= 6 -> " + ge6);
+
+ for (int i = 0; i < idx.size(); i++) {
+ System.out.print(idx.getCode(i) + (i + 1 == idx.size() ? "\n" : " "));
+ }
+
+ System.out.println("count(<5) = " + idx.count(Op.LT, 5));
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongQueryTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongQueryTest.java
new file mode 100644
index 0000000..928356c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongQueryTest.java
@@ -0,0 +1,327 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class HBPIndexLongQueryTest {
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<HBPIndexLong> indexList, byte[] encoded_result) {
+
+ long[] block_data = new long[remainder];
+ System.arraycopy(data, block_index * block_size, block_data, 0, remainder);
+
+ long min_value = Long.MAX_VALUE;
+ long max_value = Long.MIN_VALUE;
+ for (long value : block_data) {
+ if (value < min_value) {
+ min_value = value;
+ }
+ if (value > max_value) {
+ max_value = value;
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ block_data[i] -= min_value;
+ }
+
+ long2Bytes(min_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_value - min_value);
+
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ HBPIndexLong idx = new HBPIndexLong(bw, block_data);
+ indexList.add(idx);
+
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<HBPIndexLong> indexList, int[] result, int[] result_length) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ HBPIndexLong idx = indexList.get(block_index);
+
+ BitSet bitset_result = idx.select(HBPIndexLong.Op.GT, 0);
+
+ for (int i = 0; i < bitset_result.length(); i++) {
+ if (bitset_result.get(i)) {
+ result[result_length[0]] = i + (block_index * block_size);
+ result_length[0]++;
+ }
+ }
+
+ return encode_pos;
+
+ }
+
+ public static int Encoder(long[] data, int block_size, ArrayList<HBPIndexLong> indexList, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, indexList, encoded_result);
+ }
+
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, indexList,
+ encoded_result);
+
+ return encode_pos;
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<HBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length);
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "hbp_query.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ indexList.clear();
+
+ length = Encoder(data2_arr, block_size, indexList, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (HBPIndexLong idx : indexList) {
+ compressed_size += idx.segments * (idx.k + 1) * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encoded_result, indexList);
+
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "HBP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongTest.java
new file mode 100644
index 0000000..fffb052
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/HBPIndexLongTest.java
@@ -0,0 +1,325 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class HBPIndexLongTest {
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<HBPIndexLong> indexList, byte[] encoded_result) {
+
+ long[] block_data = new long[remainder];
+ System.arraycopy(data, block_index * block_size, block_data, 0, remainder);
+
+ long min_value = Long.MAX_VALUE;
+ long max_value = Long.MIN_VALUE;
+ for (long value : block_data) {
+ if (value < min_value) {
+ min_value = value;
+ }
+ if (value > max_value) {
+ max_value = value;
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ block_data[i] -= min_value;
+ }
+
+ long2Bytes(min_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_value - min_value);
+
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ HBPIndexLong idx = new HBPIndexLong(bw, block_data);
+ indexList.add(idx);
+
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<HBPIndexLong> indexList, long[] data) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ HBPIndexLong idx = indexList.get(block_index);
+
+ for (int i = 0; i < remainder; i++) {
+ long value = idx.getCode(i);
+
+ data[block_index * block_size + i] = value + min_value;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static int Encoder(long[] data, int block_size, ArrayList<HBPIndexLong> indexList, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, indexList, encoded_result);
+ }
+
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, indexList,
+ encoded_result);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result, ArrayList<HBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, data);
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D://github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "hbp.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<HBPIndexLong> indexList = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ indexList.clear();
+
+ length = Encoder(data2_arr, block_size, indexList, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (HBPIndexLong idx : indexList) {
+ compressed_size += idx.segments * (idx.k + 1) * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result, indexList);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "HBP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectEqual.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectEqual.java
new file mode 100644
index 0000000..339d231
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectEqual.java
@@ -0,0 +1,444 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class ParquetSelectEqual {
+
+ // -------------------------
+ // 辅助函数
+ // -------------------------
+ public static int popcount(long x) { return Long.bitCount(x); }
+
+ public static long pext64(long src, long mask) {
+ return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+ Long.bitCount(mask) == 64 ? src : // 全掩码优化
+ Long.bitCount(mask) == 1 ? (src & mask) != 0 ? 1 : 0 : // 单比特掩码优化
+ pext64Impl(src, mask); // 原始实现
+ }
+
+ private static long pext64Impl(long src, long mask) {
+ long out = 0L;
+ long outPos = 0L;
+ long m = mask;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> bitIndex) & 1L;
+ out |= (bit << outPos);
+ outPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long pdep64(long src, long mask) {
+ return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+ Long.bitCount(mask) == 64 ? src : // 全掩码优化
+ Long.bitCount(mask) == 1 ? (src & 1) != 0 ? mask : 0 : // 单比特掩码优化
+ pdep64Impl(src, mask); // 原始实现
+ }
+
+ private static long pdep64Impl(long src, long mask) {
+ long out = 0L;
+ long m = mask;
+ long srcPos = 0L;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> srcPos) & 1L;
+ if (bit != 0L) out |= (1L << bitIndex);
+ srcPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long extend64(long bitmap, long mask) {
+ long low = pdep64(bitmap, mask);
+ long high = pdep64(bitmap, mask - 1L);
+ return high - low;
+ }
+
+ public static long selectWord64(long valuesWord, long bitmap, long mask) {
+ long extended = extend64(bitmap, mask);
+ return pext64(valuesWord, extended);
+ }
+
+ // -------------------------
+ // 优化的按块打包函数
+ // -------------------------
+
+ /** 计算最小能表示 range 所需的 bit 数 */
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+ /**
+ * Pack integer array into blocks of long[] words using k bits per value.
+ * Each block contains up to blockSize values.
+ */
+ public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+ if (k <= 0 || k > 32) k = 32;
+ int fieldsPerWord = 64 / k;
+ int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+ int numBlocks = (values.length + blockSize - 1) / blockSize;
+
+ long[][] blocks = new long[numBlocks][];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+
+ for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+ int blockStart = blockIdx * blockSize;
+ int blockEnd = Math.min(blockStart + blockSize, values.length);
+ int blockValues = blockEnd - blockStart;
+
+ long[] blockWords = new long[wordsPerBlock];
+
+ for (int i = 0; i < blockValues; i++) {
+ int globalIndex = blockStart + i;
+ int widx = i / fieldsPerWord;
+ int pos = (i % fieldsPerWord) * k;
+ long v = ((long) values[globalIndex]) & mask;
+ blockWords[widx] |= (v << pos);
+ }
+
+ blocks[blockIdx] = blockWords;
+ }
+
+ return blocks;
+ }
+
+ // compute maskHigh: highest bit position for each k-bit field (used to PEXT MSB)
+ public static long computeMaskHigh(int k) {
+ if (k <= 0 || k > 64) return 0;
+ long m = 0L;
+ int fields = 64 / k;
+ for (int i = 0; i < fields; i++) {
+ int pos = i * k + (k - 1);
+ if (pos < 64) {
+ m |= (1L << pos);
+ }
+ }
+ return m;
+ }
+
+ // -------------------------
+ // 优化的等于查询函数
+ // -------------------------
+// public static int[] queryEqualFromBlocks(long[][] blocks, int totalValues, int k, int offset, int targetValue, int blockSize) {
+// if (k <= 0) throw new IllegalArgumentException("k must be > 0");
+// int fieldsPerWord = 64 / k;
+// long fieldMask = (k >= 64) ? ~0L : ((1L << k) - 1L);
+//
+// // 预计算每个块中的值数量
+// int numBlocks = blocks.length;
+// int[] valuesPerBlock = new int[numBlocks];
+// for (int i = 0; i < numBlocks - 1; i++) {
+// valuesPerBlock[i] = blockSize;
+// }
+// valuesPerBlock[numBlocks - 1] = totalValues - (numBlocks - 1) * blockSize;
+//
+// // 使用更高效的直接位操作而不是selectWord64
+// int[] temp = new int[totalValues];
+// int outLen = 0;
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// long[] block = blocks[blockIdx];
+// int valuesInBlock = valuesPerBlock[blockIdx];
+// int wordsInBlock = (valuesInBlock + fieldsPerWord - 1) / fieldsPerWord;
+//
+// for (int widx = 0; widx < wordsInBlock; widx++) {
+// long word = block[widx];
+//
+// // 直接提取字段而不是使用selectWord64
+// for (int b = 0; b < fieldsPerWord; b++) {
+// int localIndex = widx * fieldsPerWord + b;
+// if (localIndex >= valuesInBlock) break;
+//
+// int globalIndex = blockIdx * blockSize + localIndex;
+// // 使用直接位移和掩码操作提取值
+// int val = (int) ((word >>> (b * k)) & fieldMask);
+// int actual = val + offset;
+// if (actual == targetValue) { // 修改为等于比较
+// temp[outLen++] = globalIndex;
+// }
+// }
+// }
+// }
+//
+// // 只返回实际需要的部分
+// if (outLen == totalValues) {
+// return temp; // 所有值都满足条件
+// }
+//
+// int[] res = new int[outLen];
+// System.arraycopy(temp, 0, res, 0, outLen);
+// return res;
+// }
+
+// public static int[] queryEqualFromBlocks(
+// long[][] packedBlocks,
+// int n,
+// int k,
+// int min,
+// int upper, // 传入比较上限
+// int blockSize) {
+//
+// List<Integer> hits = new ArrayList<>();
+// int numBlocks = packedBlocks.length;
+//
+// for (int b = 0; b < numBlocks; b++) {
+// long[] block = packedBlocks[b];
+// int startIdx = b * blockSize;
+// int blockCount = Math.min(blockSize, n - startIdx);
+//
+// for (int i = 0; i < blockCount; i++) {
+// long val = extractKbitValue(block, i, k); // shifted value
+// long original = val + (long) min;
+// // 关键改动：小于比较
+// if (original == upper) {
+// hits.add(startIdx + i);
+// }
+// }
+// }
+//
+// // 转为 int[]
+// int[] out = new int[hits.size()];
+// for (int i = 0; i < hits.size(); i++) {
+// out[i] = hits.get(i);
+// }
+// return out;
+// }
+ public static long[] queryEqualFromBlocks(
+ long[][] packedBlocks,
+ int n,
+ int k,
+ int min,
+ int upper, // 传入比较上限
+ int blockSize) {
+
+ List<Long> hits = new ArrayList<>(); // 改为存储Long值
+ int numBlocks = packedBlocks.length;
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // 提取压缩值
+ long original = val + (long) min; // 恢复原始值
+
+ // 关键改动：返回原始值而不是索引
+ if (original == upper) {
+ hits.add(original); // 添加原始值到结果列表
+ }
+ }
+ }
+
+ // 转为 long[]
+ long[] out = new long[hits.size()];
+ for (int i = 0; i < hits.size(); i++) {
+ out[i] = hits.get(i);
+ }
+ return out;
+ }
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ // 跨 word 边界
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+
+ // -------------------------
+ // 辅助 I/O 函数
+ // -------------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) return 0;
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) return "";
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) return fileName;
+ return fileName.substring(0, dotIndex);
+ }
+
+ // -------------------------
+ // 优化的 main 函数
+ // -------------------------
+ public static void main(String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ // 修改为等于查询的目标值
+ HashMap<String, Integer> queryEqualValue = new HashMap<>();
+ queryEqualValue.put("Bird-migration", 2600000);
+ queryEqualValue.put("Bitcoin-price", 170000000);
+ queryEqualValue.put("City-temp", 700);
+ queryEqualValue.put("Dewpoint-temp", 9600);
+ queryEqualValue.put("IR-bio-temp", -200);
+ queryEqualValue.put("PM10-dust", 2000);
+ queryEqualValue.put("Stocks-DE", 90000);
+ queryEqualValue.put("Stocks-UK", 30000);
+ queryEqualValue.put("Stocks-USA", 6000);
+ queryEqualValue.put("Wind-Speed", 60);
+ queryEqualValue.put("Wine-Tasting", 10);
+ queryEqualValue.put("Arade4", 10000000);
+ queryEqualValue.put("EPM-Education", 200);
+ queryEqualValue.put("POI-lat", 0);
+ queryEqualValue.put("Gov10", 100000);
+
+ int repeatTime = 100;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_equal.csv"; // 修改输出文件名
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 预热JVM
+ packToBlocks(shifted, k, block_size);
+ queryEqualFromBlocks(packToBlocks(shifted, k, block_size), n, k, min,
+ queryEqualValue.getOrDefault(datasetName, 0) * max_mul, block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ int target = queryEqualValue.getOrDefault(datasetName, 0) * max_mul;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ long[] hits = queryEqualFromBlocks(packedBlocks, n, k, min, target, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreater.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreater.java
new file mode 100644
index 0000000..454f1e9
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreater.java
@@ -0,0 +1,451 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+// 把下面的方法放到与你的 main 同一个类里（作为 static 方法），或放入一个工具类并在 main 中调用。
+import java.util.ArrayList;
+import java.util.List;
+public class ParquetSelectGreater {
+
+// // -------------------------
+// // 辅助函数
+// // -------------------------
+// public static int popcount(long x) { return Long.bitCount(x); }
+//
+// public static long pext64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & mask) != 0 ? 1 : 0 : // 单比特掩码优化
+// pext64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pext64Impl(long src, long mask) {
+// long out = 0L;
+// long outPos = 0L;
+// long m = mask;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> bitIndex) & 1L;
+// out |= (bit << outPos);
+// outPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long pdep64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & 1) != 0 ? mask : 0 : // 单比特掩码优化
+// pdep64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pdep64Impl(long src, long mask) {
+// long out = 0L;
+// long m = mask;
+// long srcPos = 0L;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> srcPos) & 1L;
+// if (bit != 0L) out |= (1L << bitIndex);
+// srcPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long extend64(long bitmap, long mask) {
+// long low = pdep64(bitmap, mask);
+// long high = pdep64(bitmap, mask - 1L);
+// return high - low;
+// }
+//
+// public static long selectWord64(long valuesWord, long bitmap, long mask) {
+// long extended = extend64(bitmap, mask);
+// return pext64(valuesWord, extended);
+// }
+//
+// // -------------------------
+// // 优化的按块打包函数
+// // -------------------------
+
+ /** 计算最小能表示 range 所需的 bit 数 */
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+// /**
+// * Pack integer array into blocks of long[] words using k bits per value.
+// * Each block contains up to blockSize values.
+// */
+// public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+// if (k <= 0 || k > 32) k = 32;
+// int fieldsPerWord = 64 / k;
+// int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+// int numBlocks = (values.length + blockSize - 1) / blockSize;
+//
+// long[][] blocks = new long[numBlocks][];
+// long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// int blockStart = blockIdx * blockSize;
+// int blockEnd = Math.min(blockStart + blockSize, values.length);
+// int blockValues = blockEnd - blockStart;
+//
+// long[] blockWords = new long[wordsPerBlock];
+//
+// for (int i = 0; i < blockValues; i++) {
+// int globalIndex = blockStart + i;
+// int widx = i / fieldsPerWord;
+// int pos = (i % fieldsPerWord) * k;
+// long v = ((long) values[globalIndex]) & mask;
+// blockWords[widx] |= (v << pos);
+// }
+//
+// blocks[blockIdx] = blockWords;
+// }
+//
+// return blocks;
+// }
+//
+// // compute maskHigh: highest bit position for each k-bit field (used to PEXT MSB)
+// public static long computeMaskHigh(int k) {
+// if (k <= 0 || k > 64) return 0;
+// long m = 0L;
+// int fields = 64 / k;
+// for (int i = 0; i < fields; i++) {
+// int pos = i * k + (k - 1);
+// if (pos < 64) {
+// m |= (1L << pos);
+// }
+// }
+// return m;
+// }
+//
+// // -------------------------
+// // 优化的查询函数，使用直接位操作而不是selectWord64
+// // -------------------------
+// public static int[] queryGreaterThanFromBlocks(long[][] blocks, int totalValues, int k, int offset, int lowerBound, int blockSize) {
+// if (k <= 0) throw new IllegalArgumentException("k must be > 0");
+// int fieldsPerWord = 64 / k;
+// long fieldMask = (k >= 64) ? ~0L : ((1L << k) - 1L);
+//
+// // 预计算每个块中的值数量
+// int numBlocks = blocks.length;
+// int[] valuesPerBlock = new int[numBlocks];
+// for (int i = 0; i < numBlocks - 1; i++) {
+// valuesPerBlock[i] = blockSize;
+// }
+// valuesPerBlock[numBlocks - 1] = totalValues - (numBlocks - 1) * blockSize;
+//
+// // 使用更高效的直接位操作而不是selectWord64
+// int[] temp = new int[totalValues];
+// int outLen = 0;
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// long[] block = blocks[blockIdx];
+// int valuesInBlock = valuesPerBlock[blockIdx];
+// int wordsInBlock = (valuesInBlock + fieldsPerWord - 1) / fieldsPerWord;
+//
+// for (int widx = 0; widx < wordsInBlock; widx++) {
+// long word = block[widx];
+//
+// // 直接提取字段而不是使用selectWord64
+// for (int b = 0; b < fieldsPerWord; b++) {
+// int localIndex = widx * fieldsPerWord + b;
+// if (localIndex >= valuesInBlock) break;
+//
+// int globalIndex = blockIdx * blockSize + localIndex;
+// // 使用直接位移和掩码操作提取值
+// int val = (int) ((word >>> (b * k)) & fieldMask);
+// int actual = val + offset;
+// if (actual > lowerBound) {
+// temp[outLen++] = globalIndex;
+// }
+// }
+// }
+// }
+//
+// // 只返回实际需要的部分
+// if (outLen == totalValues) {
+// return temp; // 所有值都满足条件
+// }
+//
+// int[] res = new int[outLen];
+// System.arraycopy(temp, 0, res, 0, outLen);
+// return res;
+// }
+//
+ // -------------------------
+ // 辅助 I/O 函数
+ // -------------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) return 0;
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) return "";
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) return fileName;
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ public static long[][] packToBlocks(int[] shifted, int k, int blockSize) {
+ int n = shifted.length;
+ int numBlocks = (n + blockSize - 1) / blockSize;
+ long[][] blocks = new long[numBlocks][];
+ for (int b = 0; b < numBlocks; b++) {
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+ long bits = (long) blockCount * k;
+ int words = (int) ((bits + 63) / 64);
+ long[] buf = new long[words];
+ // 按值逐位写入（little-endian bit packing，每个 value 的最低位放在更低的 bit）
+ for (int i = 0; i < blockCount; i++) {
+ int value = shifted[startIdx + i];
+ long bitPos = (long) i * k;
+ for (int bit = 0; bit < k; bit++) {
+ int bval = (value >>> bit) & 1;
+ if (bval != 0) {
+ setBit(buf, bitPos + bit, 1);
+ } // 若为0可跳过（buf 默认 0）
+ }
+ }
+ blocks[b] = buf;
+ }
+ return blocks;
+ }
+
+ public static int[] queryGreaterThanFromBlocks(long[][] packedBlocks, int n, int k, int min, int lower, int blockSize) {
+ List<Integer> hits = new ArrayList<>();
+ int numBlocks = packedBlocks.length;
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // shifted value
+ long original = val + (long) min;
+ if (original > lower) {
+ hits.add(startIdx + i);
+ }
+ }
+ }
+ // 转为 int[]
+ int[] out = new int[hits.size()];
+ for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
+ return out;
+ }
+
+
+ /* ------- 辅助位操作（逐位最慢实现） ------- */
+
+ private static int getBit(long[] words, long bitIndex) {
+ int w = (int) (bitIndex >>> 6); // /64
+ int off = (int) (bitIndex & 63L);
+ return (int) ((words[w] >>> off) & 1L);
+ }
+
+ private static void setBit(long[] words, long bitIndex, int v) {
+ int w = (int) (bitIndex >>> 6);
+ int off = (int) (bitIndex & 63L);
+ if (v == 1) {
+ words[w] |= (1L << off);
+ } else {
+ words[w] &= ~(1L << off);
+ }
+ }
+
+ // 从单个 block 的 bit-packed long[] 中提取第 idx 个 k-bit 值（little-endian packing）
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ // 跨 word 边界
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+
+ // -------------------------
+ // 优化的 main 函数
+ // -------------------------
+ public static void main(String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+
+ int repeatTime = 200;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_greater.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 预热JVM
+// packToBlocks(shifted, k, block_size);
+ queryGreaterThanFromBlocks(packToBlocks(shifted, k, block_size), n, k, min,
+ queryRange.getOrDefault(datasetName, 0), block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+// }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ int lower = queryRange.getOrDefault(datasetName, 0);
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int[] hits = queryGreaterThanFromBlocks(packedBlocks, n, k, min, lower, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreaterLess.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreaterLess.java
new file mode 100644
index 0000000..f443b0b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectGreaterLess.java
@@ -0,0 +1,433 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class ParquetSelectGreaterLess {
+
+ // -------------------------
+ // 辅助函数
+ // -------------------------
+ public static int popcount(long x) { return Long.bitCount(x); }
+
+ public static long pext64(long src, long mask) {
+ return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+ Long.bitCount(mask) == 64 ? src : // 全掩码优化
+ Long.bitCount(mask) == 1 ? (src & mask) != 0 ? 1 : 0 : // 单比特掩码优化
+ pext64Impl(src, mask); // 原始实现
+ }
+
+ private static long pext64Impl(long src, long mask) {
+ long out = 0L;
+ long outPos = 0L;
+ long m = mask;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> bitIndex) & 1L;
+ out |= (bit << outPos);
+ outPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long pdep64(long src, long mask) {
+ return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+ Long.bitCount(mask) == 64 ? src : // 全掩码优化
+ Long.bitCount(mask) == 1 ? (src & 1) != 0 ? mask : 0 : // 单比特掩码优化
+ pdep64Impl(src, mask); // 原始实现
+ }
+
+ private static long pdep64Impl(long src, long mask) {
+ long out = 0L;
+ long m = mask;
+ long srcPos = 0L;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> srcPos) & 1L;
+ if (bit != 0L) out |= (1L << bitIndex);
+ srcPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long extend64(long bitmap, long mask) {
+ long low = pdep64(bitmap, mask);
+ long high = pdep64(bitmap, mask - 1L);
+ return high - low;
+ }
+
+ public static long selectWord64(long valuesWord, long bitmap, long mask) {
+ long extended = extend64(bitmap, mask);
+ return pext64(valuesWord, extended);
+ }
+
+ // -------------------------
+ // 优化的按块打包函数
+ // -------------------------
+
+ /** 计算最小能表示 range 所需的 bit 数 */
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+ /**
+ * Pack integer array into blocks of long[] words using k bits per value.
+ * Each block contains up to blockSize values.
+ */
+ public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+ if (k <= 0 || k > 32) k = 32;
+ int fieldsPerWord = 64 / k;
+ int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+ int numBlocks = (values.length + blockSize - 1) / blockSize;
+
+ long[][] blocks = new long[numBlocks][];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+
+ for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+ int blockStart = blockIdx * blockSize;
+ int blockEnd = Math.min(blockStart + blockSize, values.length);
+ int blockValues = blockEnd - blockStart;
+
+ long[] blockWords = new long[wordsPerBlock];
+
+ for (int i = 0; i < blockValues; i++) {
+ int globalIndex = blockStart + i;
+ int widx = i / fieldsPerWord;
+ int pos = (i % fieldsPerWord) * k;
+ long v = ((long) values[globalIndex]) & mask;
+ blockWords[widx] |= (v << pos);
+ }
+
+ blocks[blockIdx] = blockWords;
+ }
+
+ return blocks;
+ }
+
+ // compute maskHigh: highest bit position for each k-bit field (used to PEXT MSB)
+ public static long computeMaskHigh(int k) {
+ if (k <= 0 || k > 64) return 0;
+ long m = 0L;
+ int fields = 64 / k;
+ for (int i = 0; i < fields; i++) {
+ int pos = i * k + (k - 1);
+ if (pos < 64) {
+ m |= (1L << pos);
+ }
+ }
+ return m;
+ }
+
+ // -------------------------
+ // 优化的区间查询函数（大于下界且小于上界）
+ // -------------------------
+// public static int[] queryRangeFromBlocks(long[][] blocks, int totalValues, int k, int offset,
+// int lowerBound, int upperBound, int blockSize) {
+// if (k <= 0) throw new IllegalArgumentException("k must be > 0");
+// int fieldsPerWord = 64 / k;
+// long fieldMask = (k >= 64) ? ~0L : ((1L << k) - 1L);
+//
+// // 预计算每个块中的值数量
+// int numBlocks = blocks.length;
+// int[] valuesPerBlock = new int[numBlocks];
+// for (int i = 0; i < numBlocks - 1; i++) {
+// valuesPerBlock[i] = blockSize;
+// }
+// valuesPerBlock[numBlocks - 1] = totalValues - (numBlocks - 1) * blockSize;
+//
+// // 使用更高效的直接位操作而不是selectWord64
+// int[] temp = new int[totalValues];
+// int outLen = 0;
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// long[] block = blocks[blockIdx];
+// int valuesInBlock = valuesPerBlock[blockIdx];
+// int wordsInBlock = (valuesInBlock + fieldsPerWord - 1) / fieldsPerWord;
+//
+// for (int widx = 0; widx < wordsInBlock; widx++) {
+// long word = block[widx];
+//
+// // 直接提取字段而不是使用selectWord64
+// for (int b = 0; b < fieldsPerWord; b++) {
+// int localIndex = widx * fieldsPerWord + b;
+// if (localIndex >= valuesInBlock) break;
+//
+// int globalIndex = blockIdx * blockSize + localIndex;
+// // 使用直接位移和掩码操作提取值
+// int val = (int) ((word >>> (b * k)) & fieldMask);
+// int actual = val + offset;
+// // 修改为区间查询条件（大于下界且小于上界）
+// if (actual > lowerBound && actual < upperBound) {
+// temp[outLen++] = globalIndex;
+// }
+// }
+// }
+// }
+//
+// // 只返回实际需要的部分
+// if (outLen == totalValues) {
+// return temp; // 所有值都满足条件
+// }
+//
+// int[] res = new int[outLen];
+// System.arraycopy(temp, 0, res, 0, outLen);
+// return res;
+// }
+
+ public static int[] queryRangeFromBlocks(
+ long[][] packedBlocks,
+ int n,
+ int k,
+ int min,
+ int upper, // 传入比较上限
+ int lower, // 传入比较下限
+ int blockSize) {
+
+ List<Integer> hits = new ArrayList<>();
+ int numBlocks = packedBlocks.length;
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // shifted value
+ long original = val + (long) min;
+ // 关键改动：小于比较
+ if (original < upper && original > lower) {
+ hits.add(startIdx + i);
+ }
+ }
+ }
+
+ // 转为 int[]
+ int[] out = new int[hits.size()];
+ for (int i = 0; i < hits.size(); i++) {
+ out[i] = hits.get(i);
+ }
+ return out;
+ }
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ // 跨 word 边界
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+
+ // -------------------------
+ // 辅助 I/O 函数
+ // -------------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) return 0;
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) return "";
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) return fileName;
+ return fileName.substring(0, dotIndex);
+ }
+
+ // -------------------------
+ // 优化的 main 函数
+ // -------------------------
+ public static void main(String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ // 定义区间查询的上下界
+ HashMap<String, Integer> queryGreaterRange = new HashMap<>();
+ queryGreaterRange.put("Bird-migration", 2500000);
+ queryGreaterRange.put("Bitcoin-price", 160000000);
+ queryGreaterRange.put("City-temp", 480);
+ queryGreaterRange.put("Dewpoint-temp", 9500);
+ queryGreaterRange.put("IR-bio-temp", -300);
+ queryGreaterRange.put("PM10-dust", 1000);
+ queryGreaterRange.put("Stocks-DE", 40000);
+ queryGreaterRange.put("Stocks-UK", 20000);
+ queryGreaterRange.put("Stocks-USA", 5000);
+ queryGreaterRange.put("Wind-Speed", 50);
+ queryGreaterRange.put("Wine-Tasting", 0);
+ queryGreaterRange.put("Arade4", 10000000);
+ queryGreaterRange.put("EPM-Education", 200);
+ queryGreaterRange.put("POI-lat", 0);
+ queryGreaterRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap<>();
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 11000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 10);
+ queryLessRange.put("Gov10", 110000);
+
+ int repeatTime = 100;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_greater_less.csv"; // 修改输出文件名
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 获取当前数据集的上下界
+ int lowerBound = queryGreaterRange.getOrDefault(datasetName, 0) * max_mul;
+ int upperBound = queryLessRange.getOrDefault(datasetName, 0) * max_mul;
+
+ // 预热JVM
+ packToBlocks(shifted, k, block_size);
+ queryRangeFromBlocks(packToBlocks(shifted, k, block_size), n, k, min,
+ lowerBound, upperBound, block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int[] hits = queryRangeFromBlocks(packedBlocks, n, k, min, lowerBound, upperBound, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLess.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLess.java
new file mode 100644
index 0000000..2554d68
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLess.java
@@ -0,0 +1,342 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+public class ParquetSelectLess {
+
+ // -----------------------
+ // 软件 PEXT / PDEP 实现
+ // -----------------------
+ public static long pext64(long src, long mask) {
+ long out = 0L;
+ long outPos = 0L;
+ long m = mask;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> bitIndex) & 1L;
+ out |= (bit << outPos);
+ outPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long pdep64(long src, long mask) {
+ long out = 0L;
+ long m = mask;
+ long srcPos = 0L;
+ while (m != 0L) {
+ long lowest = m & -m;
+ int bitIndex = Long.numberOfTrailingZeros(lowest);
+ long bit = (src >>> srcPos) & 1L;
+ if (bit != 0L) out |= (1L << bitIndex);
+ srcPos++;
+ m &= m - 1;
+ }
+ return out;
+ }
+
+ public static long extend64(long bitmap, long mask) {
+ long low = pdep64(bitmap, mask);
+ long high = pdep64(bitmap, mask - 1L);
+ return high - low;
+ }
+
+ public static long selectWord64(long valuesWord, long bitmap, long mask) {
+ long extended = extend64(bitmap, mask);
+ return pext64(valuesWord, extended);
+ }
+
+ public static long computeMaskHigh(int k) {
+ long m = 0L;
+ for (int i = 0; i * k + (k - 1) < 64; i++) {
+ int pos = i * k + (k - 1);
+ m |= (1L << pos);
+ }
+ return m;
+ }
+
+ // -----------------------
+ // 按块处理的 less than 查询实现
+ // -----------------------
+
+ /**
+ * Pack integer array into blocks of long[] words using k bits per value.
+ * Each block contains up to blockSize values.
+ * Returns a 2D array where first dimension is block index and second is words in block.
+ */
+ public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+ if (k <= 0 || k > 32) k = 32;
+ int fieldsPerWord = 64 / k;
+ int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+ int numBlocks = (values.length + blockSize - 1) / blockSize;
+
+ long[][] blocks = new long[numBlocks][wordsPerBlock];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+
+ for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+ int blockStart = blockIdx * blockSize;
+ int blockEnd = Math.min(blockStart + blockSize, values.length);
+ int blockValues = blockEnd - blockStart;
+
+ for (int i = 0; i < blockValues; i++) {
+ int globalIndex = blockStart + i;
+ int widx = i / fieldsPerWord;
+ int pos = (i % fieldsPerWord) * k;
+ long v = ((long) values[globalIndex]) & mask;
+ blocks[blockIdx][widx] |= (v << pos);
+ }
+ }
+
+ return blocks;
+ }
+
+ /**
+ * Query less than from packed blocks
+ * 优化版本：使用直接位操作而不是 selectWord64 提高性能
+ */
+ public static int[] queryLessThanFromBlocks(
+ long[][] packedBlocks,
+ int n,
+ int k,
+ int min,
+ int upper, // 传入比较上限
+ int blockSize) {
+
+ List<Integer> hits = new ArrayList<>();
+ int numBlocks = packedBlocks.length;
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // shifted value
+ long original = val + (long) min;
+ // 关键改动：小于比较
+ if (original < upper) {
+ hits.add(startIdx + i);
+ }
+ }
+ }
+
+ // 转为 int[]
+ int[] out = new int[hits.size()];
+ for (int i = 0; i < hits.size(); i++) {
+ out[i] = hits.get(i);
+ }
+ return out;
+ }
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ // 跨 word 边界
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+
+ // -----------------------
+ // 工具函数
+ // -----------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+ // -----------------------
+ // testQuery()：修改为使用二维数组
+ // -----------------------
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 200;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_less.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(!queryRange.containsKey(datasetName))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+
+ long[][] packedBlocks = null;
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // Encoding benchmark
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // Calculate compressed size
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int upper = queryRange.getOrDefault(datasetName, 0);
+ int[] hits = queryLessThanFromBlocks(packedBlocks, n, k, min, upper, block_size);
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("Compression ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLessPart.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLessPart.java
new file mode 100644
index 0000000..f67d080
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectLessPart.java
@@ -0,0 +1,538 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+// 把下面的方法放到与你的 main 同一个类里（作为 static 方法），或放入一个工具类并在 main 中调用。
+
+
+public class ParquetSelectLessPart {
+
+// // -------------------------
+// // 辅助函数
+// // -------------------------
+// public static int popcount(long x) { return Long.bitCount(x); }
+//
+// public static long pext64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & mask) != 0 ? 1 : 0 : // 单比特掩码优化
+// pext64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pext64Impl(long src, long mask) {
+// long out = 0L;
+// long outPos = 0L;
+// long m = mask;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> bitIndex) & 1L;
+// out |= (bit << outPos);
+// outPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long pdep64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & 1) != 0 ? mask : 0 : // 单比特掩码优化
+// pdep64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pdep64Impl(long src, long mask) {
+// long out = 0L;
+// long m = mask;
+// long srcPos = 0L;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> srcPos) & 1L;
+// if (bit != 0L) out |= (1L << bitIndex);
+// srcPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long extend64(long bitmap, long mask) {
+// long low = pdep64(bitmap, mask);
+// long high = pdep64(bitmap, mask - 1L);
+// return high - low;
+// }
+//
+// public static long selectWord64(long valuesWord, long bitmap, long mask) {
+// long extended = extend64(bitmap, mask);
+// return pext64(valuesWord, extended);
+// }
+//
+// // -------------------------
+// // 优化的按块打包函数
+// // -------------------------
+
+ /** 计算最小能表示 range 所需的 bit 数 */
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+// /**
+// * Pack integer array into blocks of long[] words using k bits per value.
+// * Each block contains up to blockSize values.
+// */
+// public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+// if (k <= 0 || k > 32) k = 32;
+// int fieldsPerWord = 64 / k;
+// int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+// int numBlocks = (values.length + blockSize - 1) / blockSize;
+//
+// long[][] blocks = new long[numBlocks][];
+// long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// int blockStart = blockIdx * blockSize;
+// int blockEnd = Math.min(blockStart + blockSize, values.length);
+// int blockValues = blockEnd - blockStart;
+//
+// long[] blockWords = new long[wordsPerBlock];
+//
+// for (int i = 0; i < blockValues; i++) {
+// int globalIndex = blockStart + i;
+// int widx = i / fieldsPerWord;
+// int pos = (i % fieldsPerWord) * k;
+// long v = ((long) values[globalIndex]) & mask;
+// blockWords[widx] |= (v << pos);
+// }
+//
+// blocks[blockIdx] = blockWords;
+// }
+//
+// return blocks;
+// }
+//
+// // compute maskHigh: highest bit position for each k-bit field (used to PEXT MSB)
+// public static long computeMaskHigh(int k) {
+// if (k <= 0 || k > 64) return 0;
+// long m = 0L;
+// int fields = 64 / k;
+// for (int i = 0; i < fields; i++) {
+// int pos = i * k + (k - 1);
+// if (pos < 64) {
+// m |= (1L << pos);
+// }
+// }
+// return m;
+// }
+//
+// // -------------------------
+// // 优化的查询函数，使用直接位操作而不是selectWord64
+// // -------------------------
+// public static int[] queryGreaterThanFromBlocks(long[][] blocks, int totalValues, int k, int offset, int lowerBound, int blockSize) {
+// if (k <= 0) throw new IllegalArgumentException("k must be > 0");
+// int fieldsPerWord = 64 / k;
+// long fieldMask = (k >= 64) ? ~0L : ((1L << k) - 1L);
+//
+// // 预计算每个块中的值数量
+// int numBlocks = blocks.length;
+// int[] valuesPerBlock = new int[numBlocks];
+// for (int i = 0; i < numBlocks - 1; i++) {
+// valuesPerBlock[i] = blockSize;
+// }
+// valuesPerBlock[numBlocks - 1] = totalValues - (numBlocks - 1) * blockSize;
+//
+// // 使用更高效的直接位操作而不是selectWord64
+// int[] temp = new int[totalValues];
+// int outLen = 0;
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// long[] block = blocks[blockIdx];
+// int valuesInBlock = valuesPerBlock[blockIdx];
+// int wordsInBlock = (valuesInBlock + fieldsPerWord - 1) / fieldsPerWord;
+//
+// for (int widx = 0; widx < wordsInBlock; widx++) {
+// long word = block[widx];
+//
+// // 直接提取字段而不是使用selectWord64
+// for (int b = 0; b < fieldsPerWord; b++) {
+// int localIndex = widx * fieldsPerWord + b;
+// if (localIndex >= valuesInBlock) break;
+//
+// int globalIndex = blockIdx * blockSize + localIndex;
+// // 使用直接位移和掩码操作提取值
+// int val = (int) ((word >>> (b * k)) & fieldMask);
+// int actual = val + offset;
+// if (actual > lowerBound) {
+// temp[outLen++] = globalIndex;
+// }
+// }
+// }
+// }
+//
+// // 只返回实际需要的部分
+// if (outLen == totalValues) {
+// return temp; // 所有值都满足条件
+// }
+//
+// int[] res = new int[outLen];
+// System.arraycopy(temp, 0, res, 0, outLen);
+// return res;
+// }
+//
+ // -------------------------
+ // 辅助 I/O 函数
+ // -------------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) return 0;
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) return "";
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) return fileName;
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ public static long[][] packToBlocks(int[] shifted, int k, int blockSize) {
+ int n = shifted.length;
+ int numBlocks = (n + blockSize - 1) / blockSize;
+ long[][] blocks = new long[numBlocks][];
+ for (int b = 0; b < numBlocks; b++) {
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+ long bits = (long) blockCount * k;
+ int words = (int) ((bits + 63) / 64);
+ long[] buf = new long[words];
+ // 按值逐位写入（little-endian bit packing，每个 value 的最低位放在更低的 bit）
+ for (int i = 0; i < blockCount; i++) {
+ int value = shifted[startIdx + i];
+ long bitPos = (long) i * k;
+ for (int bit = 0; bit < k; bit++) {
+ int bval = (value >>> bit) & 1;
+ if (bval != 0) {
+ setBit(buf, bitPos + bit, 1);
+ } // 若为0可跳过（buf 默认 0）
+ }
+ }
+ blocks[b] = buf;
+ }
+ return blocks;
+ }
+
+ public static int[] queryGreaterThanFromBlocks(long[][] packedBlocks, int n, int k, int min, int lower, int blockSize) {
+ List<Integer> hits = new ArrayList<>();
+ int numBlocks = packedBlocks.length;
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // shifted value
+ long original = val + (long) min;
+ if (original > lower) {
+ hits.add(startIdx + i);
+ }
+ }
+ }
+ // 转为 int[]
+ int[] out = new int[hits.size()];
+ for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
+ return out;
+ }
+ public static int[] queryTwoColumnSerialRange(
+ long[][] packedA, int kA, int minA, int upperA, int lowerA,
+ long[][] packedB, int kB, int minB, int upperB, int lowerB,
+ int n, int blockSize) {
+
+
+// 第一步：对列 A 进行过滤，得到一个 BitSet（或 boolean[]）
+ BitSet passA = queryRangeBitmapFromBlocks(packedA, n, kA, minA, upperA, lowerA, blockSize);
+
+
+// 第二步：仅对 passA 为 true 的索引在列 B 上做过滤
+ List<Integer> hits = new ArrayList<>();
+
+
+// 遍历所有通过 A 的索引（使用 BitSet.nextSetBit 加速遍历稀疏位图）
+ int idx = passA.nextSetBit(0);
+ while (idx >= 0 && idx < n) {
+ long val = extractKbitValue(packedB, idx, kB, blockSize);
+ long original = val + (long) minB;
+ if (original < upperB && original > lowerB) {
+ hits.add(idx);
+ }
+ idx = passA.nextSetBit(idx + 1);
+ }
+
+
+// 转为 int[] 返回
+ int[] out = new int[hits.size()];
+ for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
+ return out;
+ }
+ public static BitSet queryRangeBitmapFromBlocks(
+ long[][] packedBlocks,
+ int n,
+ int k,
+ int min,
+ int upper,
+ int lower,
+ int blockSize) {
+
+
+ BitSet pass = new BitSet(n);
+ int numBlocks = packedBlocks.length;
+
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k);
+ long original = val + (long) min;
+ if (original < upper && original > lower) {
+ pass.set(startIdx + i);
+ }
+ }
+ }
+
+
+ return pass;
+ }
+ private static long extractKbitValue(long[][] packedBlocks, int globalIdx, int k, int blockSize) {
+ int blockId = globalIdx / blockSize;
+ int inBlockIdx = globalIdx % blockSize;
+ long[] block = packedBlocks[blockId];
+ return extractKbitValue(block, inBlockIdx, k);
+ }
+ /* ------- 辅助位操作（逐位最慢实现） ------- */
+
+ private static int getBit(long[] words, long bitIndex) {
+ int w = (int) (bitIndex >>> 6); // /64
+ int off = (int) (bitIndex & 63L);
+ return (int) ((words[w] >>> off) & 1L);
+ }
+
+ private static void setBit(long[] words, long bitIndex, int v) {
+ int w = (int) (bitIndex >>> 6);
+ int off = (int) (bitIndex & 63L);
+ if (v == 1) {
+ words[w] |= (1L << off);
+ } else {
+ words[w] &= ~(1L << off);
+ }
+ }
+
+ // 从单个 block 的 bit-packed long[] 中提取第 idx 个 k-bit 值（little-endian packing）
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+ // -------------------------
+ // 优化的 main 函数
+ // -------------------------
+ public static void main(String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+
+ int repeatTime = 200;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_less_parts.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+// --- 在读完 data2_arr（长度为 n）之后，替换下面这段代码 ---
+// 将单列对半分成两列（每列长度 n2 = n/2）
+ int mid = n / 2; // 向下取整
+ int n2 = mid; // 作为新“记录数”用于两列并行/串行逻辑
+ if (n2 == 0) {
+ System.err.println("Too few rows to split into two columns for dataset " + datasetName);
+ continue;
+ }
+
+// 构造两列原始整数数组
+ int[] colA_raw = Arrays.copyOfRange(data2_arr, 0, n2);
+ int[] colB_raw = Arrays.copyOfRange(data2_arr, n2, n2 + n2); // 若 n 为奇数，最后一个元素被忽略
+
+// 为 A 列计算 min/max/bitwidth kA
+ int minA = Integer.MAX_VALUE, maxA = Integer.MIN_VALUE;
+ for (int v : colA_raw) { if (v < minA) minA = v; if (v > maxA) maxA = v; }
+ long rangeA = (long) maxA - (long) minA;
+ int kA = neededBitsForRange(rangeA);
+ if (kA < 1) kA = 1; if (kA > 32) kA = 32;
+
+// 为 B 列计算 min/max/bitwidth kB
+ int minB = Integer.MAX_VALUE, maxB = Integer.MIN_VALUE;
+ for (int v : colB_raw) { if (v < minB) minB = v; if (v > maxB) maxB = v; }
+ long rangeB = (long) maxB - (long) minB;
+ int kB = neededBitsForRange(rangeB);
+ if (kB < 1) kB = 1; if (kB > 32) kB = 32;
+
+// 构造 shifted 数组（减去对应的 min，使得所有值非负）
+ int[] shiftedA = new int[n2];
+ int[] shiftedB = new int[n2];
+ for (int i = 0; i < n2; i++) {
+ shiftedA[i] = colA_raw[i] - minA;
+ shiftedB[i] = colB_raw[i] - minB;
+ }
+
+// 预热与编码基准（只做一次 pack 以测编码耗时的近似值，沿用你的 repeatTime 用法）
+ long encodeTimeA = 0, encodeTimeB = 0;
+ long s_enc = System.nanoTime();
+ long[][] packedA = packToBlocks(shiftedA, kA, block_size);
+ long e_enc = System.nanoTime();
+ encodeTimeA += ((e_enc - s_enc) / repeatTime); // 保持与原代码同样的除法
+
+ s_enc = System.nanoTime();
+ long[][] packedB = packToBlocks(shiftedB, kB, block_size);
+ e_enc = System.nanoTime();
+ encodeTimeB += ((e_enc - s_enc) / repeatTime);
+
+// 计算合并后的压缩大小（字节）
+ double compressed_size = 0;
+ for (long[] block : packedA) compressed_size += block.length * Long.BYTES;
+ for (long[] block : packedB) compressed_size += block.length * Long.BYTES;
+
+// 压缩比（仍按原逻辑，注意现在总点数为 n2 * 2 或者你想按每列单独算）
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n2 * Integer.BYTES * 2); // 两列合计占用的原始大小
+ } else {
+ ratioTmp = compressed_size / (double) (n2 * Long.BYTES * 2);
+ }
+
+ System.out.println("Querying (two-column serial) ...");
+
+// 使用同一个阈值（dataset 的 queryRange）作为 lower（你可以改成不同阈值）
+ int queryLower = queryRange.getOrDefault(datasetName, 0);
+// 为了兼容原来的判断 (original < upper && original > lower)，这里我们把 upper 设为 Integer.MAX_VALUE
+ int queryUpper = Integer.MAX_VALUE;
+
+// 计时：重复多次调用两列串行过滤
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int[] hits = queryTwoColumnSerialRange(
+ packedA, kA, minA, queryUpper, queryLower,
+ packedB, kB, minB, queryUpper, queryLower,
+ n2, block_size);
+ // hits 未做后续处理，仅用于模拟查询负载
+ }
+ long e = System.nanoTime();
+ long decodeTime = ((e - s) / repeatTime);
+
+// 写出记录（注意把 Points 更新成 n2）
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto (split2cols)",
+ String.valueOf((encodeTimeA + encodeTimeB)),
+ String.valueOf(decodeTime),
+ String.valueOf(n2),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("kA (bits): " + kA + " kB (bits): " + kB + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+
+
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectMax.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectMax.java
new file mode 100644
index 0000000..2c15581
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/ParquetSelectMax.java
@@ -0,0 +1,806 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+// 把下面的方法放到与你的 main 同一个类里（作为 static 方法），或放入一个工具类并在 main 中调用。
+
+
+public class ParquetSelectMax {
+
+// // -------------------------
+// // 辅助函数
+// // -------------------------
+// public static int popcount(long x) { return Long.bitCount(x); }
+//
+// public static long pext64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & mask) != 0 ? 1 : 0 : // 单比特掩码优化
+// pext64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pext64Impl(long src, long mask) {
+// long out = 0L;
+// long outPos = 0L;
+// long m = mask;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> bitIndex) & 1L;
+// out |= (bit << outPos);
+// outPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long pdep64(long src, long mask) {
+// return Long.bitCount(mask) == 0 ? 0 : // 添加边界检查
+// Long.bitCount(mask) == 64 ? src : // 全掩码优化
+// Long.bitCount(mask) == 1 ? (src & 1) != 0 ? mask : 0 : // 单比特掩码优化
+// pdep64Impl(src, mask); // 原始实现
+// }
+//
+// private static long pdep64Impl(long src, long mask) {
+// long out = 0L;
+// long m = mask;
+// long srcPos = 0L;
+// while (m != 0L) {
+// long lowest = m & -m;
+// int bitIndex = Long.numberOfTrailingZeros(lowest);
+// long bit = (src >>> srcPos) & 1L;
+// if (bit != 0L) out |= (1L << bitIndex);
+// srcPos++;
+// m &= m - 1;
+// }
+// return out;
+// }
+//
+// public static long extend64(long bitmap, long mask) {
+// long low = pdep64(bitmap, mask);
+// long high = pdep64(bitmap, mask - 1L);
+// return high - low;
+// }
+//
+// public static long selectWord64(long valuesWord, long bitmap, long mask) {
+// long extended = extend64(bitmap, mask);
+// return pext64(valuesWord, extended);
+// }
+//
+// // -------------------------
+// // 优化的按块打包函数
+// // -------------------------
+
+ /** 计算最小能表示 range 所需的 bit 数 */
+ public static int neededBitsForRange(long range) {
+ if (range <= 0) return 1;
+ return 64 - Long.numberOfLeadingZeros(range);
+ }
+
+// /**
+// * Pack integer array into blocks of long[] words using k bits per value.
+// * Each block contains up to blockSize values.
+// */
+// public static long[][] packToBlocks(int[] values, int k, int blockSize) {
+// if (k <= 0 || k > 32) k = 32;
+// int fieldsPerWord = 64 / k;
+// int wordsPerBlock = (blockSize + fieldsPerWord - 1) / fieldsPerWord;
+// int numBlocks = (values.length + blockSize - 1) / blockSize;
+//
+// long[][] blocks = new long[numBlocks][];
+// long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// int blockStart = blockIdx * blockSize;
+// int blockEnd = Math.min(blockStart + blockSize, values.length);
+// int blockValues = blockEnd - blockStart;
+//
+// long[] blockWords = new long[wordsPerBlock];
+//
+// for (int i = 0; i < blockValues; i++) {
+// int globalIndex = blockStart + i;
+// int widx = i / fieldsPerWord;
+// int pos = (i % fieldsPerWord) * k;
+// long v = ((long) values[globalIndex]) & mask;
+// blockWords[widx] |= (v << pos);
+// }
+//
+// blocks[blockIdx] = blockWords;
+// }
+//
+// return blocks;
+// }
+//
+// // compute maskHigh: highest bit position for each k-bit field (used to PEXT MSB)
+// public static long computeMaskHigh(int k) {
+// if (k <= 0 || k > 64) return 0;
+// long m = 0L;
+// int fields = 64 / k;
+// for (int i = 0; i < fields; i++) {
+// int pos = i * k + (k - 1);
+// if (pos < 64) {
+// m |= (1L << pos);
+// }
+// }
+// return m;
+// }
+//
+// // -------------------------
+// // 优化的查询函数，使用直接位操作而不是selectWord64
+// // -------------------------
+// public static int[] queryGreaterThanFromBlocks(long[][] blocks, int totalValues, int k, int offset, int lowerBound, int blockSize) {
+// if (k <= 0) throw new IllegalArgumentException("k must be > 0");
+// int fieldsPerWord = 64 / k;
+// long fieldMask = (k >= 64) ? ~0L : ((1L << k) - 1L);
+//
+// // 预计算每个块中的值数量
+// int numBlocks = blocks.length;
+// int[] valuesPerBlock = new int[numBlocks];
+// for (int i = 0; i < numBlocks - 1; i++) {
+// valuesPerBlock[i] = blockSize;
+// }
+// valuesPerBlock[numBlocks - 1] = totalValues - (numBlocks - 1) * blockSize;
+//
+// // 使用更高效的直接位操作而不是selectWord64
+// int[] temp = new int[totalValues];
+// int outLen = 0;
+//
+// for (int blockIdx = 0; blockIdx < numBlocks; blockIdx++) {
+// long[] block = blocks[blockIdx];
+// int valuesInBlock = valuesPerBlock[blockIdx];
+// int wordsInBlock = (valuesInBlock + fieldsPerWord - 1) / fieldsPerWord;
+//
+// for (int widx = 0; widx < wordsInBlock; widx++) {
+// long word = block[widx];
+//
+// // 直接提取字段而不是使用selectWord64
+// for (int b = 0; b < fieldsPerWord; b++) {
+// int localIndex = widx * fieldsPerWord + b;
+// if (localIndex >= valuesInBlock) break;
+//
+// int globalIndex = blockIdx * blockSize + localIndex;
+// // 使用直接位移和掩码操作提取值
+// int val = (int) ((word >>> (b * k)) & fieldMask);
+// int actual = val + offset;
+// if (actual > lowerBound) {
+// temp[outLen++] = globalIndex;
+// }
+// }
+// }
+// }
+//
+// // 只返回实际需要的部分
+// if (outLen == totalValues) {
+// return temp; // 所有值都满足条件
+// }
+//
+// int[] res = new int[outLen];
+// System.arraycopy(temp, 0, res, 0, outLen);
+// return res;
+// }
+//
+ // -------------------------
+ // 辅助 I/O 函数
+ // -------------------------
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) return 0;
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) return "";
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) return fileName;
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ public static long[][] packToBlocks(int[] shifted, int k, int blockSize) {
+ int n = shifted.length;
+ int numBlocks = (n + blockSize - 1) / blockSize;
+ long[][] blocks = new long[numBlocks][];
+ for (int b = 0; b < numBlocks; b++) {
+ int startIdx = b * blockSize;
+ int blockCount = Math.min(blockSize, n - startIdx);
+ long bits = (long) blockCount * k;
+ int words = (int) ((bits + 63) / 64);
+ long[] buf = new long[words];
+ // 按值逐位写入（little-endian bit packing，每个 value 的最低位放在更低的 bit）
+ for (int i = 0; i < blockCount; i++) {
+ int value = shifted[startIdx + i];
+ long bitPos = (long) i * k;
+ for (int bit = 0; bit < k; bit++) {
+ int bval = (value >>> bit) & 1;
+ if (bval != 0) {
+ setBit(buf, bitPos + bit, 1);
+ } // 若为0可跳过（buf 默认 0）
+ }
+ }
+ blocks[b] = buf;
+ }
+ return blocks;
+ }
+
+// public static int[] queryGreaterThanFromBlocks(long[][] packedBlocks, int n, int k, int min, int lower, int blockSize) {
+// List<Integer> hits = new ArrayList<>();
+// int numBlocks = packedBlocks.length;
+// for (int b = 0; b < numBlocks; b++) {
+// long[] block = packedBlocks[b];
+// int startIdx = b * blockSize;
+// int blockCount = Math.min(blockSize, n - startIdx);
+// for (int i = 0; i < blockCount; i++) {
+// long val = extractKbitValue(block, i, k); // shifted value
+// long original = val + (long) min;
+// if (original > lower) {
+// hits.add(startIdx + i);
+// }
+// }
+// }
+// // 转为 int[]
+// int[] out = new int[hits.size()];
+// for (int i = 0; i < hits.size(); i++) out[i] = hits.get(i);
+// return out;
+// }
+public static long calculateSumFromBlocks(long[][] packedBlocks, int n, int k, int min, int blockSize) {
+ long sum = 0L;
+ int numBlocks = packedBlocks.length;
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int blockCount = Math.min(blockSize, n - b * blockSize);
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // 提取压缩值
+ int original = (int) (val + min); // 转换为原始值
+ sum += original; // 累加到总和
+ }
+ }
+
+ return sum;
+}
+ public static int findMaxFromBlocks(long[][] packedBlocks, int n, int k, int min, int blockSize) {
+ int maxVal = Integer.MIN_VALUE;
+ int numBlocks = packedBlocks.length;
+
+ for (int b = 0; b < numBlocks; b++) {
+ long[] block = packedBlocks[b];
+ int blockCount = Math.min(blockSize, n - b * blockSize);
+
+ for (int i = 0; i < blockCount; i++) {
+ long val = extractKbitValue(block, i, k); // 提取压缩值
+ int original = (int) (val + min); // 转换为原始值
+ if (original > maxVal) {
+ maxVal = original;
+ }
+ }
+ }
+
+ return maxVal;
+ }
+
+ /* ------- 辅助位操作（逐位最慢实现） ------- */
+
+ private static int getBit(long[] words, long bitIndex) {
+ int w = (int) (bitIndex >>> 6); // /64
+ int off = (int) (bitIndex & 63L);
+ return (int) ((words[w] >>> off) & 1L);
+ }
+
+ private static void setBit(long[] words, long bitIndex, int v) {
+ int w = (int) (bitIndex >>> 6);
+ int off = (int) (bitIndex & 63L);
+ if (v == 1) {
+ words[w] |= (1L << off);
+ } else {
+ words[w] &= ~(1L << off);
+ }
+ }
+
+ // 从单个 block 的 bit-packed long[] 中提取第 idx 个 k-bit 值（little-endian packing）
+ private static long extractKbitValue(long[] valuesWords, int idx, int k) {
+ long bitPos = (long) idx * (long) k;
+ int w = (int) (bitPos >>> 6);
+ int off = (int) (bitPos & 63L);
+ if (off + k <= 64) {
+ long word = valuesWords[w];
+ long mask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ return (word >>> off) & mask;
+ } else {
+ // 跨 word 边界
+ int lowBits = 64 - off;
+ long lowMask = (lowBits == 64) ? ~0L : ((1L << lowBits) - 1L);
+ long low = (valuesWords[w] >>> off) & lowMask;
+ long high = valuesWords[w + 1] & ((1L << (k - lowBits)) - 1L);
+ return (high << lowBits) | low;
+ }
+ }
+ public static int countFromBlocks(long[][] packedBlocks, int k, int blockSize) {
+ int totalCount = 0;
+
+ for (long[] block : packedBlocks) {
+ // 计算每个块中的值数量
+ // 每个块最多有 blockSize 个值，但最后一个块可能不满
+ // 我们可以通过块的总位数除以每个值的位数来计算实际值数量
+ long totalBitsInBlock = (long) block.length * 64L;
+ int valuesInBlock = (int) (totalBitsInBlock / k);
+
+ // 确保不超过块大小
+ valuesInBlock = Math.min(valuesInBlock, blockSize);
+
+ totalCount += valuesInBlock;
+ }
+
+ return totalCount;
+ }
+ // -------------------------
+ // 优化的 main 函数
+ // -------------------------
+ @Test
+ public void maxTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+
+ int repeatTime = 100;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_max.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 预热JVM
+// packToBlocks(shifted, k, block_size);
+ findMaxFromBlocks(packToBlocks(shifted, k, block_size), n, k, min, block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+// }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ int lower = queryRange.getOrDefault(datasetName, 0);
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int hits = findMaxFromBlocks(packedBlocks, n, k, min, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+
+
+ @Test
+ public void sumTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+
+ int repeatTime = 200;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_sum.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 预热JVM
+// packToBlocks(shifted, k, block_size);
+ calculateSumFromBlocks(packToBlocks(shifted, k, block_size), n, k, min, block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+// }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ int lower = queryRange.getOrDefault(datasetName, 0);
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ calculateSumFromBlocks(packedBlocks, n, k, min, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+
+ @Test
+ public void countTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/query_parquetproto/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+
+ int repeatTime = 200;
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ String outputPath = output_parent_dir + "parquetselect_query_count.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ System.err.println("No csv files under " + input_parent_dir);
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println("Dataset: " + datasetName);
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int n = data1.size();
+ int[] data2_arr = new int[n];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < n; i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // compute min/max and needed bitwidth
+ int min = Integer.MAX_VALUE, max = Integer.MIN_VALUE;
+ for (int v : data2_arr) {
+ if (v < min) min = v;
+ if (v > max) max = v;
+ }
+ long range = (long) max - (long) min;
+ int k = neededBitsForRange(range);
+ if (k < 1) k = 1;
+ if (k > 32) k = 32;
+
+ // pack values (subtract min to make non-negative)
+ int[] shifted = new int[n];
+ for (int i = 0; i < n; i++) shifted[i] = data2_arr[i] - min;
+ long[][] packedBlocks = null;
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressed_size = 0;
+
+ // 预热JVM
+// packToBlocks(shifted, k, block_size);
+ calculateSumFromBlocks(packToBlocks(shifted, k, block_size), n, k, min, block_size);
+
+ // encoding benchmark: repeatedly pack
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ packedBlocks = packToBlocks(shifted, k, block_size);
+// }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 计算压缩大小
+ for (long[] block : packedBlocks) {
+ compressed_size += block.length * Long.BYTES;
+ }
+
+ double ratioTmp;
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (n * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (n * Long.BYTES);
+ }
+
+ System.out.println("Querying...");
+
+ int lower = queryRange.getOrDefault(datasetName, 0);
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int hits = countFromBlocks( packedBlocks,k, block_size);
+ // hits not used further here, just to simulate query work
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "ParquetSelect-proto",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(n),
+ String.valueOf((long) compressed_size),
+ String.valueOf(ratioTmp)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("k (bits): " + k + " compressed bytes: " + (long) compressed_size + " ratio: " + ratioTmp);
+ }
+
+ writer.close();
+ System.out.println("Done. Results written to " + outputPath);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongOnSortedTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongOnSortedTest.java
new file mode 100644
index 0000000..348a9b7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongOnSortedTest.java
@@ -0,0 +1,863 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+public class RLEBPLongOnSortedTest {
+
+ public static int getBitWith(int num) {
+ if (num == 0)
+ return 1;
+ else
+ return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static int getBitWith(long num) {
+ if (num == 0)
+ return 1;
+ else
+ return 64 - Long.numberOfLeadingZeros(num);
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ ArrayList<Integer> repeat_count) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+ for (int j = base; j < end; j++) {
+
+ long integer = ts_block[j];
+ if (integer < value_delta_min)
+ value_delta_min = integer;
+ if (integer > value_delta_max) {
+ value_delta_max = integer;
+ }
+ }
+ long pre_delta = ts_block[i * block_size] - value_delta_min;
+ int pre_count = 1;
+
+ min_delta[0] = (value_delta_min);
+ int repeat_i = 0;
+ int ts_block_delta_i = 0;
+ for (int j = base + 1; j < end; j++) {
+ long delta = ts_block[j] - value_delta_min;
+ if (delta == pre_delta) {
+ pre_count++;
+ } else {
+ if (pre_count > 7) {
+ repeat_count.add(repeat_i);
+ repeat_count.add(pre_count);
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ } else {
+ for (int k = 0; k < pre_count; k++) {
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ }
+ }
+ pre_count = 1;
+ repeat_i = j - i * block_size;
+ }
+ pre_delta = delta;
+
+ }
+ for (int j = 0; j < pre_count; j++) {
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ }
+ min_delta[1] = (ts_block_delta_i);
+ min_delta[2] = (value_delta_max - value_delta_min);
+ long[] new_ts_block_delta = new long[ts_block_delta_i];
+ System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
+
+ return new_ts_block_delta;
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list) {
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0];// cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+ // cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+
+ public static int EncodeBits(long num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list) {
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0];// cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (int) (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+ // cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+
+ private static int BOSBlockEncoderImprove(long[] ts_block, int block_i, int block_size, int remaining,
+ int encode_pos, byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ long[] min_delta = new long[3];
+ long[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, init_block_size, remaining, min_delta,
+ repeat_count);
+
+ long max_delta_value = min_delta[2];
+
+ // int2Bytes(min_delta[0], encode_pos, cur_byte);
+ // encode_pos += 4;
+
+ long2Bytes(min_delta[1], encode_pos, cur_byte);
+ encode_pos += 8;
+
+ int size = repeat_count.size();
+ intByte2Bytes(size, encode_pos, cur_byte);
+ encode_pos += 1;
+
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ if (size != 0) {
+ int bit_width_init = getBitWith(init_block_size - 1);
+ for (int repeat_count_v : repeat_count) {
+ encode_pos = EncodeBits(repeat_count_v, bit_width_init, encode_pos, cur_byte, bit_index_list);
+ }
+ if (bit_index_list[0] != 8) {
+ bit_index_list[0] = 8;
+ encode_pos++;
+ }
+ }
+
+ int bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final, encode_pos, cur_byte);
+ encode_pos += 1;
+
+ bit_index_list[0] = 8;
+ for (long cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+ // final_normal.add(cur_value);
+ }
+ if (bit_index_list[0] != 8) {
+ encode_pos++;
+ }
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ long[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size, encode_pos, encoded_result);
+ // System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ // int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ // encode_pos += 4;
+ long2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all - start;
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size, remaining, encode_pos, encoded_result);
+
+ // int[] ts_block = new int[length_all-start];
+ // if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0,
+ // length_all - start);
+ //
+ //
+ // encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+ return encode_pos;
+ }
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; // cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static long DecodeBitsLong(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; // cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ long num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, long[] value_list, int init_block_size,
+ int block_size, int[] value_pos_arr) {
+
+ // int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ // decode_pos += 4;
+
+ long min_delta = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0] = decode_pos;
+ decode_list[1] = 8;
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ int bit_width_init = getBitWith(init_block_size - 1);
+ for (int i = 0; i < count_size; i++) {
+ int repeat_count_v = DecodeBits(encoded, bit_width_init, decode_list);
+ repeat_count.add(repeat_count_v);
+ }
+
+ if (decode_list[1] != 8) {
+ decode_list[1] = 8;
+ decode_list[0]++;
+ }
+ // repeat_count = decodeOutlier2Bytes(encoded, decode_pos,
+ // getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = decode_list[0];
+ // decode_list[1]= 8;
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ long pre_v;
+ int cur_i = 0;
+ int repeat_i = 0;
+
+ decode_list[0] = decode_pos;
+ decode_list[1] = 8;
+ for (int i = 0; i < cur_block_size; i++) {
+ pre_v = min_delta + DecodeBitsLong(encoded, bit_width_final, decode_list);
+ // value_list[value_pos_arr[0]++] = pre_v;
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i + 1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ }
+ if (decode_list[1] != 8) {
+ decode_list[1] = 8;
+ decode_list[0]++;
+ }
+
+ return decode_list[0];
+
+ }
+
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ long[] value_list = new long[length_all + block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+ // System.out.println(k);
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
+ // System.out.println(decode_pos);
+ }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ // int value_end = bytes2Integer(encoded, decode_pos, 4);
+ // decode_pos += 4;
+ long value_end = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+// String input_parent_dir = parent_dir + "dataset/";
+
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+// String outputPath = output_parent_dir + "rle_long.csv";
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/"; //""D:/encoding-subcolumn/result/";
+
+
+ String outputPath = output_parent_dir + "rle_long_on_sorted.csv";
+
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Points",
+ "Encoding Time",
+ "Decoding Time",
+ "Compressed Size",
+ "Compression Ratio",
+ "Encoding Time Sort",
+ "Decoding Time Sort",
+ "Compressed Size Sort",
+ "Compression Ratio Sort"
+ };
+ writer.writeRecord(head); // write header to output file
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // System.out.println(f);
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+ // ArrayList<Integer> data2 = new ArrayList<>();
+
+ // loader.readHeaders();
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ // String value = loader.getValues()[index];
+ data1.add(Double.valueOf(f_str));
+ // data2.add(Integer.valueOf(loader.getValues()[1]));
+ // data.add(Integer.valueOf(value));
+ }
+
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data1_arr = new long[data1.size()];
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ data1_arr[i] = i;
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ byte[] encoded_result1 = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int length1 = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = BOSEncoderImprove(data1_arr, block_size, encoded_result1);
+ length = BOSEncoderImprove(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ length += length1;
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES *2);
+
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ BOSDecoderImprove(encoded_result1);
+ BOSDecoderImprove(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ Integer[] indices = new Integer[data1_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = i;
+ }
+
+// 根据 data2_arr 的值对索引数组进行排序
+ long[] finalData2_arr = data2_arr;
+ Arrays.sort(indices, (i, j) -> Long.compare(finalData2_arr[i], finalData2_arr[j]));
+
+// 根据排序后的索引重新排列两个数组
+ long[] sortedData1 = new long[data1_arr.length];
+ long[] sortedData2 = new long[data2_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ sortedData1[i] = data1_arr[indices[i]];
+ sortedData2[i] = data2_arr[indices[i]];
+ }
+
+//// 将排序后的数组赋回原数组（可选）
+// data1_arr = sortedData1;
+// data2_arr = sortedData2;
+
+ System.out.println(max_decimal);
+ encoded_result = new byte[data2_arr.length * 8];
+ encoded_result1 = new byte[data2_arr.length * 8];
+
+ long encodeTime_sort = 0;
+ long decodeTime_sort = 0;
+ double ratio_sort = 0;
+ double compressed_size_sort = 0;
+
+ int length_sort = 0;
+ int length1_sort = 0;
+
+ long s_sort = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = BOSEncoderImprove(sortedData1, block_size, encoded_result1);
+ length = BOSEncoderImprove(sortedData2, block_size, encoded_result);
+ }
+
+ long e_sort = System.nanoTime();
+ encodeTime_sort += ((e_sort - s_sort) / repeatTime);
+ length += length1;
+ compressed_size_sort += length;
+
+ double ratioTmp_sort;
+
+ ratioTmp_sort = compressed_size_sort / (double) (data1.size() * Long.BYTES*2);
+
+ ratio_sort += ratioTmp_sort;
+
+ System.out.println("Decode");
+
+ long[] data1_arr_decoded_sort = new long[data2_arr.length];
+ long[] data2_arr_decoded_sort = new long[data2_arr.length];
+
+ s_sort = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ BOSDecoderImprove(encoded_result1);
+ BOSDecoderImprove(encoded_result);
+ }
+
+ e_sort = System.nanoTime();
+ decodeTime_sort += ((e_sort - s_sort) / repeatTime);
+
+// for (int i = 0; i < data2_arr_decoded_sort.length; i++) {
+// assertEquals(sortedData2[i], data2_arr_decoded_sort[i]);
+// }
+
+ String[] record = {
+ datasetName,
+ "RLE",
+ String.valueOf(data1.size()),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio),
+ String.valueOf(encodeTime_sort),
+ String.valueOf(decodeTime_sort),
+ String.valueOf(compressed_size_sort),
+ String.valueOf(ratio_sort),
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongTest.java
new file mode 100644
index 0000000..b0b839d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPLongTest.java
@@ -0,0 +1,726 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+public class RLEBPLongTest {
+
+ private static final int BIT_IO_STEP = 4;
+
+ public static int getBitWith(int num) {
+ if (num == 0)
+ return 1;
+ else
+ return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static int getBitWith(long num) {
+ if (num == 0)
+ return 1;
+ else
+ return 64 - Long.numberOfLeadingZeros(num);
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta,
+ ArrayList<Integer> repeat_count) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+ for (int j = base; j < end; j++) {
+
+ long integer = ts_block[j];
+ if (integer < value_delta_min)
+ value_delta_min = integer;
+ if (integer > value_delta_max) {
+ value_delta_max = integer;
+ }
+ }
+ long pre_delta = ts_block[i * block_size] - value_delta_min;
+ int pre_count = 1;
+
+ min_delta[0] = (value_delta_min);
+ int repeat_i = 0;
+ int ts_block_delta_i = 0;
+ for (int j = base + 1; j < end; j++) {
+ long delta = ts_block[j] - value_delta_min;
+ if (delta == pre_delta) {
+ pre_count++;
+ } else {
+ if (pre_count > 7) {
+ repeat_count.add(repeat_i);
+ repeat_count.add(pre_count);
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ } else {
+ for (int k = 0; k < pre_count; k++) {
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ }
+ }
+ pre_count = 1;
+ repeat_i = j - i * block_size;
+ }
+ pre_delta = delta;
+
+ }
+ for (int j = 0; j < pre_count; j++) {
+ ts_block_delta[ts_block_delta_i] = pre_delta;
+ ts_block_delta_i++;
+ }
+ min_delta[1] = (ts_block_delta_i);
+ min_delta[2] = (value_delta_max - value_delta_min);
+ long[] new_ts_block_delta = new long[ts_block_delta_i];
+ System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
+
+ return new_ts_block_delta;
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list) {
+ int bit_index = bit_index_list[0];
+
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
+
+ bit_index = available_bits - bits_to_write;
+
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index);
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+ return encode_pos;
+ }
+
+ public static int EncodeBits(long num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list) {
+ int bit_index = bit_index_list[0];
+
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
+
+ bit_index = available_bits - bits_to_write;
+
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (int) (num >> (remaining_bits - bits_to_write)) & mask;
+
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index);
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+ return encode_pos;
+ }
+
+ private static int BOSBlockEncoderImprove(long[] ts_block, int block_i, int block_size, int remaining,
+ int encode_pos, byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ long[] min_delta = new long[3];
+ long[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, init_block_size, remaining, min_delta,
+ repeat_count);
+
+ long max_delta_value = min_delta[2];
+
+ long2Bytes(min_delta[1], encode_pos, cur_byte);
+ encode_pos += 8;
+
+ int size = repeat_count.size();
+ intByte2Bytes(size, encode_pos, cur_byte);
+ encode_pos += 1;
+
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ if (size != 0) {
+ int bit_width_init = getBitWith(init_block_size - 1);
+ for (int repeat_count_v : repeat_count) {
+ encode_pos = EncodeBits(repeat_count_v, bit_width_init, encode_pos, cur_byte, bit_index_list);
+ }
+ if (bit_index_list[0] != 8) {
+ bit_index_list[0] = 8;
+ encode_pos++;
+ }
+ }
+
+ int bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final, encode_pos, cur_byte);
+ encode_pos += 1;
+
+ bit_index_list[0] = 8;
+ for (long cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+ }
+ if (bit_index_list[0] != 8) {
+ encode_pos++;
+ }
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ long[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ long2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all - start;
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size, remaining, encode_pos, encoded_result);
+
+ }
+
+ return encode_pos;
+ }
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ num = (num << bits_to_read) | bits;
+
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static long DecodeBitsLong(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1];
+ int remaining_bits = bit_width;
+ long num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ num = (num << bits_to_read) | bits;
+
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, long[] value_list, int init_block_size,
+ int block_size, int[] value_pos_arr) {
+
+ long min_delta = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0] = decode_pos;
+ decode_list[1] = 8;
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ int bit_width_init = getBitWith(init_block_size - 1);
+ for (int i = 0; i < count_size; i++) {
+ int repeat_count_v = DecodeBits(encoded, bit_width_init, decode_list);
+ repeat_count.add(repeat_count_v);
+ }
+
+ if (decode_list[1] != 8) {
+ decode_list[1] = 8;
+ decode_list[0]++;
+ }
+ decode_pos = decode_list[0];
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ long pre_v;
+ int cur_i = 0;
+ int repeat_i = 0;
+
+ decode_list[0] = decode_pos;
+ decode_list[1] = 8;
+ for (int i = 0; i < cur_block_size; i++) {
+ pre_v = min_delta + DecodeBitsLong(encoded, bit_width_final, decode_list);
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i + 1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ }
+ if (decode_list[1] != 8) {
+ decode_list[1] = 8;
+ decode_list[0]++;
+ }
+
+ return decode_list[0];
+
+ }
+
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ long[] value_list = new long[length_all + block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr); }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ long value_end = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, remain_length, value_pos_arr);
+ }
+ }
+
+ public static long[] decodeToLongArray(byte[] encoded) {
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ long[] value_list = new long[length_all + block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+ decode_pos =
+ BOSBlockDecoderImprove(
+ encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
+ }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ long value_end = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ decode_pos =
+ BOSBlockDecoderImprove(
+ encoded, decode_pos, value_list, block_size, remain_length, value_pos_arr);
+ }
+ return Arrays.copyOf(value_list, length_all);
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "rle_long.csv";
+
+ int block_size = 256;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BOSEncoderImprove(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ BOSDecoderImprove(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "RLE",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPTest.java
index 7fdf262..d82424e 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBPTest.java
@@ -16,8 +16,6 @@
import java.util.List;
import java.util.stream.Stream;
-import static java.lang.Math.pow;
-
public class RLEBPTest {
public static int getBitWith(int num) {
@@ -81,16 +79,12 @@
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values.get(valueIdx) << (32 - leftBit));
leftSize -= leftBit;
@@ -99,20 +93,15 @@
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values.get(valueIdx) << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values.get(valueIdx) >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -128,23 +117,16 @@
public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
result_list.add((int) (buffer >>> (totalBits - width)));
valueIdx++;
@@ -171,7 +153,7 @@
ArrayList<Integer> result_list = new ArrayList<>();
int block_num = (block_size - 1) / 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list);
decode_pos += bit_width;
}
@@ -520,29 +502,22 @@
int encode_pos,
byte[] cur_byte,
int[] bit_index_list) {
- // 找到要插入的位的索引
- int bit_index = bit_index_list[0];// cur_byte[encode_pos + 1];
+ int bit_index = bit_index_list[0];
- // 计算数值的起始位位置
int remaining_bits = bit_width;
while (remaining_bits > 0) {
- // 计算在当前字节中可以使用的位数
int available_bits = bit_index;
int bits_to_write = Math.min(available_bits, remaining_bits);
- // 更新 bit_index
bit_index = available_bits - bits_to_write;
- // 计算要写入的位的掩码和数值
int mask = (1 << bits_to_write) - 1;
int bits = (num >> (remaining_bits - bits_to_write)) & mask;
- // 写入到当前位置
- cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index);
cur_byte[encode_pos] |= (byte) (bits << bit_index);
- // 更新位宽和数值
remaining_bits -= bits_to_write;
if (bit_index == 0) {
bit_index = 8;
@@ -550,7 +525,6 @@
}
}
bit_index_list[0] = bit_index;
- // cur_byte[encode_pos + 1] = (byte) bit_index;
return encode_pos;
}
@@ -593,7 +567,6 @@
bit_index_list[0] = 8;
for (int cur_value : ts_block_delta) {
encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
- // final_normal.add(cur_value);
}
if (bit_index_list[0] != 8) {
encode_pos++;
@@ -652,11 +625,11 @@
int k_byte = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1_byte = (int) (k_byte % Math.pow(2, 16));
int k1 = k1_byte / 2;
int final_alpha = k1_byte % 2;
- int k2 = (int) (k_byte / pow(2, 16));
+ int k2 = (int) (k_byte / Math.pow(2, 16));
int min_delta = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
@@ -785,7 +758,7 @@
int right_outlier_i = 0;
int normal_i = 0;
int pre_v;
- // int final_k_end_value = (int) (final_k_start_value + pow(2,
+ // int final_k_end_value = (int) (final_k_start_value + Math.pow(2,
// bit_width_final));
int cur_i = 0;
@@ -884,14 +857,11 @@
int available_bits = bit_index;
int bits_to_read = Math.min(available_bits, remaining_bits);
- // 计算要读取的位的掩码
int mask = (1 << bits_to_read) - 1;
int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
- // 将读取的位合并到结果中
num = (num << bits_to_read) | bits;
- // 更新位宽和 bit_index
remaining_bits -= bits_to_read;
bit_index = available_bits - bits_to_read;
@@ -1007,15 +977,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -1037,24 +1004,18 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "rle.csv";
- int block_size = 1024;
+ int block_size = 512;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 200;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -1068,7 +1029,7 @@
"Compressed Size",
"Compression Ratio"
};
- writer.writeRecord(head); // write header to output file
+ writer.writeRecord(head);
File directory = new File(input_parent_dir);
// File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
@@ -1081,9 +1042,7 @@
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
ArrayList<Float> data1 = new ArrayList<>();
- // ArrayList<Integer> data2 = new ArrayList<>();
- // loader.readHeaders();
int max_decimal = 0;
while (loader.readRecord()) {
String f_str = loader.getValues()[0];
@@ -1094,10 +1053,7 @@
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
}
inputStream.close();
@@ -1108,7 +1064,7 @@
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
double ratio = 0;
@@ -1127,11 +1083,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -1160,131 +1112,4 @@
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "rle.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- BOSDecoderImprove(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "RLE",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEMaterializeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEMaterializeTest.java
new file mode 100644
index 0000000..ad2c069
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEMaterializeTest.java
@@ -0,0 +1,995 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Stream;
+
+import javax.management.Query;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class RLEMaterializeTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ min_delta[1] = value_delta_max;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result) {
+
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int maxBitWidth = bitWidth(min_delta[1] - min_delta[0]);
+
+ int2Bytes(maxBitWidth, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // long[] data_delta = new long[remainder];
+
+ // long min_value = Long.MAX_VALUE;
+ // long max_value = Long.MIN_VALUE;
+
+ // for (int j = 0; j < remainder; j++) {
+ // data_delta[j] = data[block_index * block_size + j];
+ // if (data_delta[j] < min_value) {
+ // min_value = data_delta[j];
+ // }
+ // if (data_delta[j] > max_value) {
+ // max_value = data_delta[j];
+ // }
+ // }
+
+ // int maxBitWidth = bitWidth(max_value);
+
+ // if (min_value < 0) {
+ // maxBitWidth = 64;
+ // }
+
+ // int2Bytes(maxBitWidth, encode_pos, encoded_result);
+ // encode_pos += 4;
+
+ int bw = bitWidth(remainder);
+
+ int rle_index = 0;
+ int[] run_length = new int[remainder];
+ long[] rle_values = new long[remainder];
+
+ long previous = data_delta[0];
+
+ for (int j = 1; j < remainder; j++) {
+ if (data_delta[j] != previous) {
+ run_length[rle_index] = j;
+ rle_values[rle_index] = previous;
+ rle_index++;
+ previous = data_delta[j];
+ }
+ }
+
+ run_length[rle_index] = remainder;
+ rle_values[rle_index] = previous;
+ rle_index++;
+
+ int2Bytes(rle_index, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result,
+ rle_index);
+
+ encode_pos = bitPacking(rle_values, maxBitWidth, encode_pos,
+ encoded_result, rle_index);
+
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int maxBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int rle_index = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int bw = bitWidth(remainder);
+
+ int[] run_length = new int[rle_index];
+ long[] rle_values = new long[rle_index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, rle_index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, maxBitWidth, rle_index, rle_values);
+
+ long[] ts_block_delta = new long[remainder];
+
+ int currentIndex = 0;
+ for (int i = 0; i < rle_index; i++) {
+ int end = run_length[i];
+ long value = rle_values[i];
+ while (currentIndex < end) {
+ ts_block_delta[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = ts_block_delta[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static void Query(byte[] encoded_result, long upper_bound, int[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ if (value < upper_bound) {
+ result[result_length[0]] = num_blocks * block_size + i;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int maxBitWidth = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int rle_index = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int bw = bitWidth(remainder);
+
+ int[] run_length = new int[rle_index];
+ long[] rle_values = new long[rle_index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, rle_index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, maxBitWidth, rle_index, rle_values);
+
+ long[] ts_block_delta = new long[remainder];
+
+ int currentIndex = 0;
+ for (int i = 0; i < rle_index; i++) {
+ int end = run_length[i];
+ long value = rle_values[i];
+ while (currentIndex < end) {
+ ts_block_delta[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+
+ for (int i = 0; i < currentIndex; i++) {
+ if (ts_block_delta[i] + min_delta[0] < upper_bound) {
+ result[result_length[0]] = block_index * block_size + i;
+ result_length[0]++;
+ }
+ }
+
+ return encode_pos;
+
+ }
+
+ public static double computeSelectivity(long len1_0, long len2_0, long halfSize, long match) {
+ double sA = Double.NaN, sB = Double.NaN, pAB = Double.NaN, lift = Double.NaN;
+
+ if (halfSize <= 0)
+ return lift;
+
+ // 基本概率
+ sA = (double) len1_0 / (double) halfSize;
+ sB = (double) len2_0 / (double) halfSize;
+ pAB = (double) match / (double) halfSize;
+
+ // lift = P(A∧B) / (P(A) P(B))，仅在分母非零时计算
+ if (sA > 0.0 && sB > 0.0) {
+ lift = pAB / (sA * sB);
+ }
+
+ return lift;
+ }
+
+ public static double phiCoefficient(long len1_0, long len2_0, long halfSize, long match) {
+ // 2x2 表格元素
+ double a = (double) match; // A ∧ B
+ double b = (double) (len1_0 - match); // A ∧ ¬B
+ double c = (double) (len2_0 - match); // ¬A ∧ B
+ double d = (double) (halfSize - (match + (len1_0 - match) + (len2_0 - match)));
+ // 等价于: d = halfSize - (a + b + c)
+
+ // 如果任何分量为负，输入可能不合法，返回 NaN
+ if (a < 0 || b < 0 || c < 0 || d < 0) {
+ return Double.NaN;
+ }
+
+ double numerator = a * d - b * c;
+ double denomTerm1 = (a + b) * (c + d);
+ double denomTerm2 = (a + c) * (b + d);
+
+ // 分母为 sqrt( denomTerm1 * denomTerm2 )
+ double denomProduct = denomTerm1 * denomTerm2;
+ if (denomProduct <= 0.0) {
+ return Double.NaN; // 避免除零或根号负数
+ }
+
+ double phi = numerator / Math.sqrt(denomProduct);
+ return phi;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = "D:/encoding-subcolumn/result/materialization/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "cstore_materialization2.csv";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 75000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ // "Encoding Time",
+ "Decoding Time",
+ "Selectivity",
+ "Phi",
+ "Points",
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++)
+ col1_data[i] = (long) (data1.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++)
+ col2_data[i] = (long) (data1.get(i + halfSize) * max_mul);
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 13];
+ byte[] encoded_result2 = new byte[col2_data.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ long tStart = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(col1_data, block_size, encoded_result1);
+ length2 = Encoder(col2_data, block_size, encoded_result2);
+ }
+
+ long tEnd = System.nanoTime();
+ encodeTime += ((tEnd - tStart) / repeatTime);
+ compressed_size += length1 + length2;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ int upper = queryRange.get(datasetName);
+
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ // warm run to avoid JIT one-time overhead bias
+ // Query(encoded_result1, upper, res1, len1);
+ // Query(encoded_result2, upper, res2, len2);
+
+ long[] data1_arr_decoded = new long[col1_data.length];
+ long[] data2_arr_decoded = new long[col2_data.length];
+
+ double selectivity = 0;
+ double phi = 0;
+ int match = 0;
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ // CompletableFuture<Void> future1 = CompletableFuture.runAsync(() -> {
+ // Query(encoded_result1, upper, res1, len1);
+ // });
+
+ // CompletableFuture<Void> future2 = CompletableFuture.runAsync(() -> {
+ // Query(encoded_result2, upper, res2, len2);
+ // });
+
+ // // 等待两个查询都完成
+ // try {
+ // CompletableFuture.allOf(future1, future2).get();
+ // } catch (InterruptedException | ExecutionException e) {
+ // e.printStackTrace();
+ // // 处理异常，可能需要中断循环或采取其他措施
+ // Thread.currentThread().interrupt(); // 重新设置中断状态
+ // break;
+ // }
+
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ long[] bits1 = new long[(halfSize + 63) / 64];
+ long[] bits2 = new long[(halfSize + 63) / 64];
+
+ // 设置bit
+ for (int i = 0; i < len1[0]; i++) {
+ int pos = res1[i];
+ bits1[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ for (int i = 0; i < len2[0]; i++) {
+ int pos = res2[i];
+ bits2[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ // 求交集并计数
+ match = 0;
+ for (int i = 0; i < bits1.length; i++) {
+ long intersection = bits1[i] & bits2[i];
+ match += Long.bitCount(intersection);
+ }
+ }
+ tEnd = System.nanoTime();
+
+ selectivity = computeSelectivity(len1[0], len2[0], halfSize, match);
+ phi = phiCoefficient(len1[0], len2[0], halfSize, match);
+ System.out.println(len1[0] + "," + len2[0]);
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+ // long[] data2_arr_decoded = new long[data2_arr.length];
+
+ // s = System.nanoTime();
+
+ // int[] result = new int[data2_arr.length];
+ // int[] result_length = new int[1];
+
+ // for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // // data2_arr_decoded = Decoder(encoded_result);
+ // Query(encoded_result, queryRange.get(datasetName), result, result_length);
+ // }
+
+ String[] record = {
+ datasetName,
+ "RLE",
+ // String.valueOf(encodeTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(selectivity),
+ String.valueOf(phi),
+ String.valueOf(totalSize)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPLongTest.java
new file mode 100644
index 0000000..183907f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPLongTest.java
@@ -0,0 +1,739 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+public class SPRINTZBPLongTest {
+
+ public static int getBitWith(int num) {
+ if (num == 0)
+ return 1;
+ else
+ return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static int getBitWith(long num) {
+ if (num == 0)
+ return 1;
+ else
+ return 64 - Long.numberOfLeadingZeros(num);
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int findMedian(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("Array is null or empty");
+ }
+ int n = arr.length;
+ return quickSelect(arr, 0, n - 1, n / 2);
+ }
+
+ private static int quickSelect(int[] arr, int left, int right, int k) {
+ if (left == right) {
+ return arr[left];
+ }
+
+ int pivotIndex = partition(arr, left, right);
+ if (k == pivotIndex) {
+ return arr[k];
+ } else if (k < pivotIndex) {
+ return quickSelect(arr, left, pivotIndex - 1, k);
+ } else {
+ return quickSelect(arr, pivotIndex + 1, right, k);
+ }
+ }
+
+ private static int partition(int[] arr, int left, int right) {
+ int pivot = arr[right];
+ int i = left;
+ for (int j = left; j < right; j++) {
+ if (arr[j] <= pivot) {
+ swap(arr, i, j);
+ i++;
+ }
+ }
+ swap(arr, i, right);
+ return i;
+ }
+
+ private static void swap(int[] arr, int i, int j) {
+ int temp = arr[i];
+ arr[i] = arr[j];
+ arr[j] = temp;
+ }
+
+ public static int zigzag(int num) {
+ if (num < 0)
+ return ((-num) << 1) - 1;
+ else
+ return num << 1;
+ }
+
+ public static long zigzag(long num) {
+ if (num < 0)
+ return ((-num) << 1) - 1;
+ else
+ return num << 1;
+ }
+
+ public static int deZigzag(int num) {
+ if (num % 2 == 0)
+ return num >> 1;
+ else
+ return -((num + 1) >> 1);
+ }
+
+ public static long deZigzag(long num) {
+ if (num % 2 == 0)
+ return num >> 1;
+ else
+ return -((num + 1) >> 1);
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8ValuesLong(ArrayList<Long> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8ValuesLong(byte[] encoded, int offset, int width, ArrayList<Long> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static int bitPackingLong(ArrayList<Long> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8ValuesLong(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ public static ArrayList<Long> decodeBitPackingLong(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Long> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8ValuesLong(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ int base = i * block_size + 1;
+ int end = i * block_size + remaining;
+ min_delta[0] = ts_block[base - 1];
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ for (int j = base; j < end; j++) {
+ long epsilon_v = ts_block[j] - ts_block[j - 1];
+ epsilon_v = zigzag(epsilon_v);
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ ts_block_delta[j - base] = epsilon_v;
+
+ }
+ for (int j = 0; j < remaining - 1; j++) {
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+
+ }
+ min_delta[1] = (value_delta_min);
+ min_delta[2] = (value_delta_max - value_delta_min);
+
+ return ts_block_delta;
+ }
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Long> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPackingLong(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0;
+ int cur_number_bits = 0;
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width;
+
+ if (cur_number_bits + bit_width >= 64) {
+ cur_remaining <<= (64 - cur_number_bits);
+ cur_bit_width = bit_width - 64 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2Bytes(cur_remaining, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (64 - cur_bit_width)) & 0xFFFFFFFFFFFFFFFFL) >> (64 - cur_bit_width));
+ }
+ cur_remaining <<= (64 - cur_number_bits);
+ long2Bytes(cur_remaining, encode_pos, encoded_result);
+ encode_pos += 8;
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Long> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Long> result_list = new ArrayList<>(
+ decodeBitPackingLong(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytes2Long(encoded, decode_pos, 8));
+ decode_pos += 8;
+ }
+
+ int cur_remaining_bits = 64;
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ long tmp = (long) (cur_number >> (64 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ long tmp = (long) (cur_number >> (64 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (64 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFFFFFFFFFL;
+ cur_remaining_bits = 64 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static int BOSBlockEncoder(long[] ts_block, int block_i, int block_size, int remaining, int encode_pos,
+ byte[] cur_byte) {
+
+ long[] min_delta = new long[3];
+ long[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+ block_size = remaining - 1;
+ long max_delta_value = min_delta[2];
+
+ long2Bytes(min_delta[0], encode_pos, cur_byte);
+ encode_pos += 8;
+ long2Bytes(min_delta[1], encode_pos, cur_byte);
+ encode_pos += 8;
+
+ int bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final, encode_pos, cur_byte);
+ encode_pos += 1;
+
+ ArrayList<Long> final_normal = new ArrayList<>();
+ for (long i : ts_block_delta) {
+ final_normal.add(i);
+ }
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final, encode_pos, cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoder(
+ long[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all, encode_pos, encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ long2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all - start;
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size, remaining, encode_pos, encoded_result);
+ }
+
+ return encode_pos;
+ }
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, long[] value_list, int block_size,
+ int[] value_pos_arr) {
+
+ ArrayList<Long> final_normal = new ArrayList<>();
+ ;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+
+ int bit_width_final = 0;
+ long value0 = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+
+ value_list[value_pos_arr[0]] = value0;
+ value_pos_arr[0]++;
+
+ long min_delta = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+
+ int normal_i = 0;
+ long pre_v = value0;
+
+ for (int i = 0; i < block_size; i++) {
+ long current_delta = final_normal.get(normal_i);
+ pre_v = deZigzag(current_delta + min_delta) + pre_v;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ long[] value_list = new long[length_all + block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size, value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ long value_end = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length--;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "sprintz_long0.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BOSEncoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ BOSDecoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "SPRINTZ",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+ writer.close();
+
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
index 0688e6a..6c6557e 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
@@ -12,13 +12,14 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Stream;
-import static java.lang.Math.pow;
-
public class SPRINTZBPTest {
+ private static final int BIT_IO_STEP = 2;
+
public static int getBitWith(int num) {
if (num == 0)
return 1;
@@ -36,7 +37,7 @@
public static int findMedian(int[] arr) {
if (arr == null || arr.length == 0) {
- throw new IllegalArgumentException("数组不能为空");
+ throw new IllegalArgumentException("Array is null or empty");
}
int n = arr.length;
return quickSelect(arr, 0, n - 1, n / 2);
@@ -122,6 +123,46 @@
return value;
}
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = 1;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = 1;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
long value = 0;
for (int i = 0; i < 4; i++) {
@@ -136,16 +177,12 @@
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values.get(valueIdx) << (32 - leftBit));
leftSize -= leftBit;
@@ -154,20 +191,15 @@
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values.get(valueIdx) << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values.get(valueIdx) >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -183,23 +215,16 @@
public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
result_list.add((int) (buffer >>> (totalBits - width)));
valueIdx++;
@@ -226,15 +251,13 @@
ArrayList<Integer> result_list = new ArrayList<>();
int block_num = (block_size - 1) / 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list);
decode_pos += bit_width;
}
return result_list;
}
- // -----------------------------------------------------------------
-
public static int[] getAbsDeltaTsBlock(
int[] ts_block,
int i,
@@ -279,11 +302,11 @@
int n_k = ts_block_delta.size();
int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
+ long cur_remaining = 0;
+ int cur_number_bits = 0;
for (int i = n_k_b * 8; i < n_k; i++) {
long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
+ int cur_bit_width = bit_width;
if (cur_number_bits + bit_width >= 32) {
cur_remaining <<= (32 - cur_number_bits);
@@ -326,7 +349,7 @@
decode_pos += 4;
}
- int cur_remaining_bits = 32; // remaining bit width of current value
+ int cur_remaining_bits = 32;
long cur_number = int_remaining.get(0);
int cur_number_i = 1;
for (int i = n_k_b * 8; i < length; i++) {
@@ -442,8 +465,8 @@
encode_pos += 4;
int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
- int left_bit_width = getBitWith(final_k_start_value);// final_left_max
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);// final_right_min
+ int left_bit_width = getBitWith(final_k_start_value);
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);
if (k1 == 0 && k2 == 0) {
bit_width_final = getBitWith(max_delta_value);
@@ -466,7 +489,7 @@
encode_pos += 1;
intByte2Bytes(right_bit_width, encode_pos, cur_byte);
encode_pos += 1;
- if (final_alpha == 0) { // 0
+ if (final_alpha == 0) {
for (int i : bitmap_outlier) {
@@ -521,29 +544,22 @@
int encode_pos,
byte[] cur_byte,
int[] bit_index_list) {
- // 找到要插入的位的索引
- int bit_index = bit_index_list[0];// cur_byte[encode_pos + 1];
+ int bit_index = bit_index_list[0];
- // 计算数值的起始位位置
int remaining_bits = bit_width;
while (remaining_bits > 0) {
- // 计算在当前字节中可以使用的位数
int available_bits = bit_index;
- int bits_to_write = Math.min(available_bits, remaining_bits);
+ int bits_to_write = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
- // 更新 bit_index
bit_index = available_bits - bits_to_write;
- // 计算要写入的位的掩码和数值
int mask = (1 << bits_to_write) - 1;
int bits = (num >> (remaining_bits - bits_to_write)) & mask;
- // 写入到当前位置
- cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index);
cur_byte[encode_pos] |= (byte) (bits << bit_index);
- // 更新位宽和数值
remaining_bits -= bits_to_write;
if (bit_index == 0) {
bit_index = 8;
@@ -743,24 +759,53 @@
}
}
+ public static int[] decodeToIntArray(byte[] encoded) {
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all + block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size, value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length--;
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ return Arrays.copyOf(value_list, length_all);
+ }
+
public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
int decode_pos = decode_pos_list[0];
- int bit_index = decode_pos_list[1]; // cur_byte[decode_pos + 1];
+ int bit_index = decode_pos_list[1];
int remaining_bits = bit_width;
int num = 0;
while (remaining_bits > 0) {
int available_bits = bit_index;
- int bits_to_read = Math.min(available_bits, remaining_bits);
+ int bits_to_read = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
- // 计算要读取的位的掩码
int mask = (1 << bits_to_read) - 1;
int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
- // 将读取的位合并到结果中
num = (num << bits_to_read) | bits;
- // 更新位宽和 bit_index
remaining_bits -= bits_to_read;
bit_index = available_bits - bits_to_read;
@@ -859,15 +904,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -889,24 +931,18 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "D://github/xjz17/subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "sprintz.csv";
int block_size = 1024;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 500;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -920,9 +956,8 @@
"Compressed Size",
"Compression Ratio"
};
- writer.writeRecord(head); // write header to output file
+ writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -934,9 +969,6 @@
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
ArrayList<Float> data1 = new ArrayList<>();
- // ArrayList<Integer> data2 = new ArrayList<>();
-
- // loader.readHeaders();
int max_decimal = 0;
while (loader.readRecord()) {
@@ -948,10 +980,7 @@
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
}
inputStream.close();
@@ -962,7 +991,7 @@
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
double ratio = 0;
@@ -981,11 +1010,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -1015,131 +1040,4 @@
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "sprintz.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- BOSDecoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "SPRINTZ",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnLongTest.java
new file mode 100644
index 0000000..0ec7667
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnLongTest.java
@@ -0,0 +1,387 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+
+public class SPRINTZSubcolumnLongTest {
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int zigzag(int num) {
+ if (num < 0)
+ return ((-num) << 1) - 1;
+ else
+ return num << 1;
+ }
+
+ public static int deZigzag(int num) {
+ if (num % 2 == 0)
+ return num >> 1;
+ else
+ return -((num + 1) >> 1);
+ }
+
+ public static long zigzag(long num) {
+ if (num < 0)
+ return ((-num) << 1) - 1;
+ else
+ return num << 1;
+ }
+
+ public static long deZigzag(long num) {
+ if (num % 2 == 0)
+ return num >> 1;
+ else
+ return -((num + 1) >> 1);
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ int base = i * block_size + 1;
+ int end = i * block_size + remaining;
+ min_delta[0] = ts_block[base - 1];
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ for (int j = base; j < end; j++) {
+ long epsilon_v = ts_block[j] - ts_block[j - 1];
+ epsilon_v = zigzag(epsilon_v);
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ ts_block_delta[j - base] = epsilon_v;
+
+ }
+ for (int j = 0; j < remaining - 1; j++) {
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+
+ }
+ min_delta[1] = (value_delta_min);
+ min_delta[2] = (value_delta_max - value_delta_min);
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size, remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ long2Bytes(min_delta[1], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder - 1; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = SubcolumnLongTest.bitWidth(maxValue);
+
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder - 1, m, block_size);
+ }
+
+ encode_pos = SubcolumnLongTest.SubcolumnEncoder(data_delta, encode_pos, encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ min_delta[1] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] data_delta = new long[remainder - 1];
+
+ encode_pos = SubcolumnLongTest.SubcolumnDecoder(encoded_result, encode_pos, data_delta, block_size);
+
+ for (int i = 0; i < remainder - 1; i++) {
+ data_delta[i] = data_delta[i] + min_delta[1];
+ }
+
+ for (int i = 0; i < remainder - 1; i++) {
+ data_delta[i] = deZigzag(data_delta[i]);
+ }
+
+ data[block_index * block_size] = min_delta[0];
+
+ for (int i = 0; i < remainder - 1; i++) {
+ data[block_index * block_size + i + 1] = data[block_index * block_size + i] + data_delta[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "sprintz_subcolumn_long.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "SPRINTZ+Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnPruneNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnPruneNewTest.java
new file mode 100644
index 0000000..a5f418b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnPruneNewTest.java
@@ -0,0 +1,398 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class SPRINTZSubcolumnPruneNewTest {
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, null, null);
+ }
+
+ public static int Encoder(
+ int[] data, int blockSize, byte[] encodedResult, long[] sprintzTime, long[] subcolumnTime) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ encodedResult[0] = (byte) (dataLength >> 24);
+ encodedResult[1] = (byte) (dataLength >> 16);
+ encodedResult[2] = (byte) (dataLength >> 8);
+ encodedResult[3] = (byte) dataLength;
+ encodePos += 4;
+
+ encodedResult[4] = (byte) (blockSize >> 24);
+ encodedResult[5] = (byte) (blockSize >> 16);
+ encodedResult[6] = (byte) (blockSize >> 8);
+ encodedResult[7] = (byte) blockSize;
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockEncoder(
+ data, i, blockSize, blockSize, encodePos, encodedResult, beta, sprintzTime, subcolumnTime);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[numBlocks * blockSize + i];
+ encodedResult[encodePos] = (byte) (value >> 24);
+ encodedResult[encodePos + 1] = (byte) (value >> 16);
+ encodedResult[encodePos + 2] = (byte) (value >> 8);
+ encodedResult[encodePos + 3] = (byte) value;
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockEncoder(
+ data, numBlocks, blockSize, remainder, encodePos, encodedResult, beta, sprintzTime, subcolumnTime);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+ int dataLength =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int blockSize =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[numBlocks * blockSize + i] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos, data);
+ }
+
+ return data;
+ }
+
+ public static int zigzag(int num) {
+ return num < 0 ? ((-num) << 1) - 1 : num << 1;
+ }
+
+ public static int deZigzag(int num) {
+ return (num % 2 == 0) ? (num >> 1) : -((num + 1) >> 1);
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] tsBlock, int blockIndex, int blockSize, int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining - 1];
+ fillAbsDeltaTsBlock(tsBlock, blockIndex, blockSize, remaining, minDelta, tsBlockDelta);
+ return tsBlockDelta;
+ }
+
+ private static void fillAbsDeltaTsBlock(
+ int[] tsBlock,
+ int blockIndex,
+ int blockSize,
+ int remaining,
+ int[] minDelta,
+ int[] tsBlockDelta) {
+ int base = blockIndex * blockSize + 1;
+ int end = blockIndex * blockSize + remaining;
+ minDelta[0] = tsBlock[base - 1];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int valueDeltaMax = Integer.MIN_VALUE;
+
+ for (int j = base; j < end; j++) {
+ int epsilon = tsBlock[j] - tsBlock[j - 1];
+ epsilon = zigzag(epsilon);
+ if (epsilon < valueDeltaMin) {
+ valueDeltaMin = epsilon;
+ }
+ if (epsilon > valueDeltaMax) {
+ valueDeltaMax = epsilon;
+ }
+ tsBlockDelta[j - base] = epsilon;
+ }
+
+ for (int j = 0; j < remaining - 1; j++) {
+ tsBlockDelta[j] = tsBlockDelta[j] - valueDeltaMin;
+ }
+ minDelta[1] = valueDeltaMin;
+ minDelta[2] = valueDeltaMax - valueDeltaMin;
+ }
+
+ public static int BlockEncoder(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta) {
+ return BlockEncoder(
+ data, blockIndex, blockSize, remainder, encodePos, encodedResult, beta, null, null);
+ }
+
+ public static int BlockEncoder(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ long[] sprintzTime,
+ long[] subcolumnTime) {
+ long sprintzStart = System.nanoTime();
+ int[] minDelta = SubcolumnPruneNewTest.borrowMinDelta3Buffer();
+ int[] dataDelta = SubcolumnPruneNewTest.borrowDataDeltaBuffer();
+ fillAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta, dataDelta);
+
+ encodedResult[encodePos] = (byte) (minDelta[0] >> 24);
+ encodedResult[encodePos + 1] = (byte) (minDelta[0] >> 16);
+ encodedResult[encodePos + 2] = (byte) (minDelta[0] >> 8);
+ encodedResult[encodePos + 3] = (byte) minDelta[0];
+ encodePos += 4;
+
+ encodedResult[encodePos] = (byte) (minDelta[1] >> 24);
+ encodedResult[encodePos + 1] = (byte) (minDelta[1] >> 16);
+ encodedResult[encodePos + 2] = (byte) (minDelta[1] >> 8);
+ encodedResult[encodePos + 3] = (byte) minDelta[1];
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int v : dataDelta) {
+ if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ int m = SubcolumnPruneNewTest.bitWidth(maxValue);
+ int[] encodingType = SubcolumnPruneNewTest.borrowEncodingTypeBuffer();
+ long sprintzEnd = System.nanoTime();
+ if (sprintzTime != null) {
+ sprintzTime[0] += (sprintzEnd - sprintzStart);
+ }
+
+ long subStart = System.nanoTime();
+ beta[0] =
+ SubcolumnPruneNewTest.Subcolumn(dataDelta, remainder - 1, m, blockSize, encodingType);
+ encodePos =
+ SubcolumnPruneNewTest.SubcolumnEncoder(
+ dataDelta,
+ remainder - 1,
+ encodePos,
+ encodedResult,
+ beta,
+ blockSize,
+ encodingType,
+ m);
+ long subEnd = System.nanoTime();
+ if (subcolumnTime != null) {
+ subcolumnTime[0] += (subEnd - subStart);
+ }
+
+ return encodePos;
+ }
+
+ public static int BlockDecoder(
+ byte[] encodedResult,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ int[] data) {
+ int[] minDelta = new int[3];
+
+ minDelta[0] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ minDelta[1] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int[] dataDelta = new int[remainder - 1];
+ encodePos = SubcolumnPruneNewTest.SubcolumnDecoder(encodedResult, encodePos, dataDelta, blockSize);
+
+ for (int i = 0; i < remainder - 1; i++) {
+ dataDelta[i] = dataDelta[i] + minDelta[1];
+ dataDelta[i] = deZigzag(dataDelta[i]);
+ }
+
+ data[blockIndex * blockSize] = minDelta[0];
+ for (int i = 0; i < remainder - 1; i++) {
+ data[blockIndex * blockSize + i + 1] = data[blockIndex * blockSize + i] + dataDelta[i];
+ }
+
+ return encodePos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parentDir = "D:/github/xjz17/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "sprintz_subcolumn_adddict_prunenew_opt2.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ "SPRINTZ Time",
+ "Subcolumn Encode Time"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, data2Arr.length * 8)];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ long sprintzTime = 0;
+ long subcolumnEncodeTime = 0;
+ double compressedSize = 0;
+ int length = 0;
+ long[] sprintzTimeArr = new long[1];
+ long[] subcolumnEncodeTimeArr = new long[1];
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ sprintzTimeArr[0] = 0;
+ subcolumnEncodeTimeArr[0] = 0;
+ length =
+ Encoder(
+ data2Arr, blockSize, encodedResult, sprintzTimeArr, subcolumnEncodeTimeArr);
+ sprintzTime += sprintzTimeArr[0];
+ subcolumnEncodeTime += subcolumnEncodeTimeArr[0];
+ }
+ long e = System.nanoTime();
+ encodeTime += (e - s) / repeatTime;
+ sprintzTime /= repeatTime;
+ subcolumnEncodeTime /= repeatTime;
+ compressedSize += length;
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s) / repeatTime;
+
+ double ratio = compressedSize / (double) (Math.max(1, data1.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SPRINTZ+Sub-columns(AddDictPruneNew-Opt2)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio),
+ String.valueOf(sprintzTime),
+ String.valueOf(subcolumnEncodeTime)
+ });
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnTest.java
index a8c8dc9..d2762ba 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZSubcolumnTest.java
@@ -151,7 +151,6 @@
int encode_pos, byte[] encoded_result, int[] beta) {
int[] min_delta = new int[3];
- // data_delta 长度为 remainder - 1
int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size, remainder, min_delta);
encoded_result[encode_pos] = (byte) (min_delta[0] >> 24);
@@ -217,15 +216,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -247,25 +243,19 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
- String outputPath = output_parent_dir + "sprintz_subcolumn.csv";
+ String outputPath = output_parent_dir + "sprintz_subcolumn2.csv";
int block_size = 512;
- int repeatTime = 100;
+ int repeatTime = 500;
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -281,7 +271,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -314,7 +303,7 @@
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -335,11 +324,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -370,133 +355,4 @@
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "sprintz_subcolumn.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "SPRINTZ+Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/Simple8bCompressionBenchmarkTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/Simple8bCompressionBenchmarkTest.java
new file mode 100644
index 0000000..965ddd2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/Simple8bCompressionBenchmarkTest.java
@@ -0,0 +1,193 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Benchmarks Simple8b (FastPFOR {@code simple8b.h}, MarkLength=true) compression ratio and
+ * encode/decode time, following the CSV workflow in {@link SubcolumnAddDictionaryTest#test0()}.
+ *
+ * <p>JavaFastPFOR does not ship Simple8b; this test uses {@link FastPForSimple8bCodec}, a Java
+ * port of the same algorithm.
+ */
+public class Simple8bCompressionBenchmarkTest {
+
+ private static final int REPEAT_WARMUP = 5;
+ private static final int REPEAT_TIMED = 200;
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testRoundTripSmall() {
+ int[] a = {0, 1, 2, 3, 100, 1000, 0xffff, 0};
+ int[] c = FastPForSimple8bCodec.encode(a);
+ int[] b = FastPForSimple8bCodec.decode(c);
+ assertArrayEquals(a, b);
+ }
+
+ @Test
+ public void testRoundTripRandom() {
+ Random rnd = new Random(42);
+ for (int n : new int[] {1, 7, 59, 120, 239, 240, 241, 500, 4000}) {
+ int[] a = new int[n];
+ for (int i = 0; i < n; i++) {
+ a[i] = rnd.nextInt(1 << 20);
+ }
+ int[] c = FastPForSimple8bCodec.encode(a);
+ int[] b = FastPForSimple8bCodec.decode(c);
+ assertArrayEquals(a, b);
+ }
+ }
+
+ @Test
+ public void testSyntheticCompressionStats() {
+ int n = 50_000;
+ int[] data = new int[n];
+ for (int i = 0; i < n; i++) {
+ data[i] = i % 17;
+ }
+ long t0 = System.nanoTime();
+ int[] compressed = null;
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ compressed = FastPForSimple8bCodec.encode(data);
+ }
+ long encNs = (System.nanoTime() - t0) / REPEAT_TIMED;
+ int[] decoded = null;
+ t0 = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ decoded = FastPForSimple8bCodec.decode(compressed);
+ }
+ long decNs = (System.nanoTime() - t0) / REPEAT_TIMED;
+ assertArrayEquals(data, decoded);
+ double rawBytes = (double) n * Integer.BYTES;
+ double compBytes = FastPForSimple8bCodec.compressedSizeBytes(compressed);
+ double ratio = compBytes / rawBytes;
+ System.out.printf(
+ "Simple8b synthetic: n=%d encode=%d ns decode=%d ns compressed=%d B ratio=%.4f%n",
+ n, encNs, decNs, (int) compBytes, ratio);
+ assertTrue("compression should not expand catastrophically on low-entropy ints", ratio < 1.2);
+ }
+
+ /**
+ * Same structure as {@link SubcolumnAddDictionaryTest#test0()}: read float CSVs, scale to int[],
+ * measure encode/decode time and compression ratio, write a result CSV. Skips if the dataset
+ * directory does not exist.
+ */
+ @Test
+ public void testCsvDatasetsIfPresent() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ File directory = new File(inputParentDir);
+ Assume.assumeTrue(
+ "Skip CSV benchmark when " + inputParentDir + " is missing", directory.isDirectory());
+
+ String outputParentDir = parentDir + "result/";
+ new File(outputParentDir).mkdirs();
+ String outputPath = outputParentDir + "simple8b_compression.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Algorithm",
+ "Encode ns",
+ "Decode ns",
+ "Points",
+ "Compressed bytes",
+ "Ratio (compressed / raw int32)"
+ });
+
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ Assume.assumeTrue(csvFiles != null);
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> floats = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, getDecimalPrecision(fStr));
+ floats.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ int[] data = new int[floats.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < floats.size(); i++) {
+ data[i] = (int) (floats.get(i) * maxMul);
+ }
+
+ for (int r = 0; r < REPEAT_WARMUP; r++) {
+ FastPForSimple8bCodec.decode(FastPForSimple8bCodec.encode(data));
+ }
+ int[] compressed = FastPForSimple8bCodec.encode(data);
+ long s = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ compressed = FastPForSimple8bCodec.encode(data);
+ }
+ long encNs = (System.nanoTime() - s) / REPEAT_TIMED;
+
+ int[] decoded = FastPForSimple8bCodec.decode(compressed);
+ s = System.nanoTime();
+ for (int r = 0; r < REPEAT_TIMED; r++) {
+ decoded = FastPForSimple8bCodec.decode(compressed);
+ }
+ long decNs = (System.nanoTime() - s) / REPEAT_TIMED;
+
+ assertArrayEquals(data, decoded);
+ int compBytes = FastPForSimple8bCodec.compressedSizeBytes(compressed);
+ double ratio = compBytes / (double) (data.length * (long) Integer.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Simple8b",
+ Long.toString(encNs),
+ Long.toString(decNs),
+ Integer.toString(data.length),
+ Integer.toString(compBytes),
+ Double.toString(ratio)
+ });
+ System.out.printf("%s Simple8b ratio=%.4f%n", datasetName, ratio);
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubColumnEncodingTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubColumnEncodingTest.java
new file mode 100644
index 0000000..916c54b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubColumnEncodingTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding;
+
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+public class SubColumnEncodingTest {
+
+ @Test
+ public void testSubColumnIntRoundTrip() throws Exception {
+ int[] values = {
+ 21, 22, 23, 23, 24, 25, 100, 101, 102, -7, -7, 0, Integer.MIN_VALUE, Integer.MAX_VALUE
+ };
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(TSEncoding.SUBCOLUMN).getEncoder(TSDataType.INT32);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (int value : values) {
+ encoder.encode(value, out);
+ }
+ encoder.flush(out);
+
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.SUBCOLUMN, TSDataType.INT32);
+ ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray());
+ for (int value : values) {
+ Assert.assertTrue(decoder.hasNext(buffer));
+ Assert.assertEquals(value, decoder.readInt(buffer));
+ }
+ Assert.assertFalse(decoder.hasNext(buffer));
+ }
+
+ @Test
+ public void testSubColumnLongRoundTrip() throws Exception {
+ long[] values = {1L, 2L, -3L, Long.MIN_VALUE, Long.MAX_VALUE};
+ Encoder encoder = TSEncodingBuilder.getEncodingBuilder(TSEncoding.SUBCOLUMN).getEncoder(TSDataType.INT64);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (long value : values) {
+ encoder.encode(value, out);
+ }
+ encoder.flush(out);
+
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.SUBCOLUMN, TSDataType.INT64);
+ ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray());
+ for (long value : values) {
+ Assert.assertTrue(decoder.hasNext(buffer));
+ Assert.assertEquals(value, decoder.readLong(buffer));
+ }
+ Assert.assertFalse(decoder.hasNext(buffer));
+ }
+
+ @Test
+ public void testSubColumnFloatRoundTrip() throws Exception {
+ float[] values = {1.23f, -4.5f, 0.001f, 100.125f, 100.125f, -0.75f};
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(TSEncoding.SUBCOLUMN).getEncoder(TSDataType.FLOAT);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ for (float value : values) {
+ encoder.encode(value, out);
+ }
+ encoder.flush(out);
+
+ Decoder decoder = Decoder.getDecoderByType(TSEncoding.SUBCOLUMN, TSDataType.FLOAT);
+ ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray());
+ for (float value : values) {
+ Assert.assertTrue(decoder.hasNext(buffer));
+ Assert.assertEquals(value, decoder.readFloat(buffer), 0.000001f);
+ }
+ Assert.assertFalse(decoder.hasNext(buffer));
+ }
+
+ @Test
+ public void testSqlEncodingNameMapsToSubColumn() {
+ Assert.assertEquals(TSEncoding.SUBCOLUMN, TSEncoding.valueOf("SubColumn".toUpperCase()));
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationCodecSupport.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationCodecSupport.java
new file mode 100644
index 0000000..c934503
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationCodecSupport.java
@@ -0,0 +1,723 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.Arrays;
+
+public final class SubcolumnAblationCodecSupport {
+
+ private static final ThreadLocal<EncodeScratch> ENCODE_SCRATCH =
+ ThreadLocal.withInitial(EncodeScratch::new);
+
+ private static final ThreadLocal<DecodeScratch> DECODE_SCRATCH =
+ ThreadLocal.withInitial(DecodeScratch::new);
+
+ public static final class EncodeScratch {
+ public final int[] dataDelta = new int[8192];
+ public final int[] bitWidthList = new int[32];
+ public final int[] subcolumnBuffer = new int[8192];
+ public final int[] runLength = new int[8192];
+ public final int[] rleValues = new int[8192];
+ public final int[] dictKeyList = new int[16];
+ public final int[] codeMap = new int[16];
+ public final int[] minDelta = new int[1];
+ public final int[] beta = new int[1];
+ }
+
+ public static final class DecodeScratch {
+ public int[] bitWidthList = new int[32];
+ public int[] encodingType = new int[32];
+ public int[] subcolumnBuffer = new int[8192];
+ public int[] runLength = new int[8192];
+ public int[] rleValues = new int[8192];
+ public int[] dictKeyList = new int[16];
+
+ public void ensureL(int l) {
+ if (bitWidthList.length < l) {
+ bitWidthList = new int[l];
+ encodingType = new int[l];
+ }
+ }
+
+ public void ensureListLength(int listLength) {
+ if (subcolumnBuffer.length < listLength) {
+ subcolumnBuffer = new int[listLength];
+ runLength = new int[listLength];
+ rleValues = new int[listLength];
+ }
+ }
+
+ public int[] ensureDict(int cardinality) {
+ if (dictKeyList.length < cardinality) {
+ dictKeyList = new int[cardinality];
+ }
+ return dictKeyList;
+ }
+ }
+
+ private SubcolumnAblationCodecSupport() {}
+
+ public static EncodeScratch encodeScratch() {
+ return ENCODE_SCRATCH.get();
+ }
+
+ public static DecodeScratch decodeScratch() {
+ return DECODE_SCRATCH.get();
+ }
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ return bitPackingAt(numbers, 0, bitWidth, encodePos, encodedResult, numValues);
+ }
+
+ public static int bitPackingAt(int[] numbers, int offset, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, offset + i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[offset + blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ public static int bitPackingWidth1At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 7) | (numbers[base + 1] << 6)
+ | (numbers[base + 2] << 5) | (numbers[base + 3] << 4)
+ | (numbers[base + 4] << 3) | (numbers[base + 5] << 2)
+ | (numbers[base + 6] << 1) | numbers[base + 7]);
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int bitPackingWidth2At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 6) | (numbers[base + 1] << 4)
+ | (numbers[base + 2] << 2) | numbers[base + 3]);
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int bitPackingWidth4At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 4) | numbers[base + 1]);
+ encodePos++;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = encodePos * 8;
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ public static int bitPackingWidth8At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ for (int i = 0; i < numValues; i++) {
+ encodedResult[encodePos++] = (byte) numbers[offset + i];
+ }
+ return encodePos;
+ }
+
+ public static int bitPackingShifted(int[] list, int listLength, int shiftAmount, int mask,
+ int bitWidth, int encodePos, byte[] encodedResult, int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ fallbackBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+ return bitPacking(fallbackBuffer, bitWidth, encodePos, encodedResult, listLength);
+ }
+
+ public static int bitPackingWidth1Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 8 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 7)
+ | (((list[i + 1] >> shiftAmount) & mask) << 6)
+ | (((list[i + 2] >> shiftAmount) & mask) << 5)
+ | (((list[i + 3] >> shiftAmount) & mask) << 4)
+ | (((list[i + 4] >> shiftAmount) & mask) << 3)
+ | (((list[i + 5] >> shiftAmount) & mask) << 2)
+ | (((list[i + 6] >> shiftAmount) & mask) << 1)
+ | ((list[i + 7] >> shiftAmount) & mask));
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int bitPackingWidth2Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 4 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 6)
+ | (((list[i + 1] >> shiftAmount) & mask) << 4)
+ | (((list[i + 2] >> shiftAmount) & mask) << 2)
+ | ((list[i + 3] >> shiftAmount) & mask));
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int bitPackingWidth4Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 2 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 4)
+ | ((list[i + 1] >> shiftAmount) & mask));
+ encodePos++;
+ i += 2;
+ }
+ if (i < listLength) {
+ int bitPos = encodePos * 8;
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ public static int bitPackingWidth8Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ for (int i = 0; i < listLength; i++) {
+ encodedResult[encodePos++] = (byte) ((list[i] >> shiftAmount) & mask);
+ }
+ return encodePos;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ if (bitWidth == 0) {
+ Arrays.fill(resultList, 0, numValues, 0);
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8(encoded, decodePos, numValues, resultList);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ public static int decodeBitPackingWidth1(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 7) & 1;
+ resultList[i + 1] = (value >>> 6) & 1;
+ resultList[i + 2] = (value >>> 5) & 1;
+ resultList[i + 3] = (value >>> 4) & 1;
+ resultList[i + 4] = (value >>> 3) & 1;
+ resultList[i + 5] = (value >>> 2) & 1;
+ resultList[i + 6] = (value >>> 1) & 1;
+ resultList[i + 7] = value & 1;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 1);
+ bitPos++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int decodeBitPackingWidth2(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 6) & 3;
+ resultList[i + 1] = (value >>> 4) & 3;
+ resultList[i + 2] = (value >>> 2) & 3;
+ resultList[i + 3] = value & 3;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 2);
+ bitPos += 2;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int decodeBitPackingWidth4(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 4) & 15;
+ resultList[i + 1] = value & 15;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ resultList[i] = bytesToInt(encoded, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ public static int decodeBitPackingWidth8(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ for (int i = 0; i < numValues; i++) {
+ resultList[i] = encoded[decodePos++] & 0xFF;
+ }
+ return decodePos;
+ }
+
+ public static int decodeBitPackingOrShifted(
+ byte[] encoded,
+ int decodePos,
+ int bitWidth,
+ int numValues,
+ int[] output,
+ int outputOffset,
+ int shiftAmount,
+ int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+
+ decodePos = decodeBitPacking(encoded, decodePos, bitWidth, numValues, fallbackBuffer);
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= fallbackBuffer[i] << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ public static int decodeBitPackingWidth1OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 7) & 1) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 6) & 1) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 5) & 1) << shiftAmount;
+ output[outputOffset + i + 3] |= ((value >>> 4) & 1) << shiftAmount;
+ output[outputOffset + i + 4] |= ((value >>> 3) & 1) << shiftAmount;
+ output[outputOffset + i + 5] |= ((value >>> 2) & 1) << shiftAmount;
+ output[outputOffset + i + 6] |= ((value >>> 1) & 1) << shiftAmount;
+ output[outputOffset + i + 7] |= (value & 1) << shiftAmount;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 1) << shiftAmount;
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int decodeBitPackingWidth2OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 6) & 3) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 4) & 3) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 2) & 3) << shiftAmount;
+ output[outputOffset + i + 3] |= (value & 3) << shiftAmount;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 2) << shiftAmount;
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ public static int decodeBitPackingWidth4OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 4) & 15) << shiftAmount;
+ output[outputOffset + i + 1] |= (value & 15) << shiftAmount;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 4) << shiftAmount;
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ public static int decodeBitPackingWidth8OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= (encoded[decodePos++] & 0xFF) << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int encodeRleRuns(
+ int[] runLength,
+ int[] rleValues,
+ int runCount,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(runLength, runLengthBitWidth, encodePos, encodedResult, runCount);
+ return bitPacking(rleValues, valueBitWidth, encodePos, encodedResult, runCount);
+ }
+
+ public static int encodeRleFromValues(
+ int[] values,
+ int offset,
+ int listLength,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = values[offset];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = values[offset + j];
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ public static int encodeRleShifted(
+ int[] list,
+ int listLength,
+ int shiftAmount,
+ int mask,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = (list[0] >> shiftAmount) & mask;
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ public static int fillAbsDeltaTsBlock(
+ int[] tsBlock,
+ int blockIndex,
+ int blockSize,
+ int remaining,
+ int[] minDelta,
+ int[] out) {
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int valueDeltaMax = Integer.MIN_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ if (current > valueDeltaMax) {
+ valueDeltaMax = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ out[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return valueDeltaMax - valueDeltaMin;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationPruneNewEngine.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationPruneNewEngine.java
new file mode 100644
index 0000000..07532e9
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAblationPruneNewEngine.java
@@ -0,0 +1,1860 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
+public final class SubcolumnAblationPruneNewEngine {
+
+ public enum Mode {
+ FULL,
+ WITHOUT_BPE,
+ WITHOUT_RLE,
+ WITHOUT_DE
+ }
+
+ private SubcolumnAblationPruneNewEngine() {}
+
+ private static boolean allowBpe(Mode mode) {
+ return mode == Mode.FULL || mode == Mode.WITHOUT_RLE || mode == Mode.WITHOUT_DE;
+ }
+
+ private static boolean allowRle(Mode mode) {
+ return mode == Mode.FULL || mode == Mode.WITHOUT_BPE || mode == Mode.WITHOUT_DE;
+ }
+
+ private static boolean allowDe(Mode mode) {
+ return mode == Mode.FULL || mode == Mode.WITHOUT_BPE || mode == Mode.WITHOUT_RLE;
+ }
+
+ private static int pickSingleBitEncodingType(int bpe, int rle, int de, Mode mode) {
+ if (allowBpe(mode) && bpe <= rle && bpe <= de) {
+ return 0;
+ }
+ if (allowRle(mode) && rle < bpe && rle <= de) {
+ return 1;
+ }
+ if (allowDe(mode)) {
+ return 2;
+ }
+ if (allowRle(mode)) {
+ return 1;
+ }
+ return 0;
+ }
+
+ private static int costOfEncodingType(int type, int bpe, int rle, int de) {
+ if (type == 0) {
+ return bpe;
+ }
+ if (type == 1) {
+ return rle;
+ }
+ return de;
+ }
+
+ private static final ThreadLocal<EncodeScratch> ENCODE_SCRATCH =
+ ThreadLocal.withInitial(EncodeScratch::new);
+
+ private static final ThreadLocal<DecodeScratch> DECODE_SCRATCH =
+ ThreadLocal.withInitial(DecodeScratch::new);
+
+ private static final class EncodeScratch {
+ private final int[] dataDelta = new int[8192];
+ private final int[] bpeCostSingle = new int[32];
+ private final int[] rleCostSingle = new int[32];
+ private final int[] deCostSingle = new int[32];
+ private final int[] encodingType = new int[32];
+ private final int[] encodingTypeTemp = new int[32];
+ private final int[] bitWidthList = new int[32];
+ private final int[] subcolumnBuffer = new int[8192];
+ private final int[] runLength = new int[8192];
+ private final int[] rleValues = new int[8192];
+ private int[] dictKeyList = new int[16];
+ private int[] codeMap = new int[16];
+ private final HashMap<Integer, Integer> dictCodeMap = new HashMap<>();
+ private int[] distinctKeys = new int[1024];
+ private int[] distinctStamps = new int[1024];
+ private int distinctEpoch = 1;
+ private final int[] minDelta = new int[1];
+ private final int[] minDelta3 = new int[3];
+ private final int[] beta = new int[1];
+ /** Flattened grouped subcolumns: group i starts at i * listLength. */
+ private final int[] groupFlat = new int[32 * 8192];
+ private final int[] groupMax = new int[32];
+ private int cachedBeta = -1;
+ private int cachedL;
+ private int cachedListLength = -1;
+
+ private int[] ensureDictKeyList(int cardinality) {
+ if (dictKeyList.length < cardinality) {
+ dictKeyList = new int[cardinality];
+ }
+ return dictKeyList;
+ }
+
+ private int[] ensureCodeMap(int valueCapacity) {
+ if (codeMap.length < valueCapacity) {
+ codeMap = new int[valueCapacity];
+ }
+ return codeMap;
+ }
+
+ private void beginDistinctSet(int expectedValues) {
+ int needed = 1;
+ while (needed < expectedValues * 4) {
+ needed <<= 1;
+ }
+ if (distinctKeys.length < needed) {
+ distinctKeys = new int[needed];
+ distinctStamps = new int[needed];
+ distinctEpoch = 1;
+ return;
+ }
+ distinctEpoch++;
+ if (distinctEpoch == 0) {
+ Arrays.fill(distinctStamps, 0);
+ distinctEpoch = 1;
+ }
+ }
+
+ private boolean addDistinctValue(int value) {
+ int mask = distinctKeys.length - 1;
+ int index = smear(value) & mask;
+ while (distinctStamps[index] == distinctEpoch) {
+ if (distinctKeys[index] == value) {
+ return false;
+ }
+ index = (index + 1) & mask;
+ }
+ distinctStamps[index] = distinctEpoch;
+ distinctKeys[index] = value;
+ return true;
+ }
+ }
+
+ private static final class DecodeScratch {
+ private int[] bitWidthList = new int[32];
+ private int[] encodingType = new int[32];
+ private int[] subcolumnBuffer = new int[8192];
+ private int[] runLength = new int[8192];
+ private int[] rleValues = new int[8192];
+ private int[] dictKeyList = new int[16];
+
+ private void ensureL(int l) {
+ if (bitWidthList.length < l) {
+ bitWidthList = new int[l];
+ encodingType = new int[l];
+ }
+ }
+
+ private void ensureListLength(int listLength) {
+ if (subcolumnBuffer.length < listLength) {
+ subcolumnBuffer = new int[listLength];
+ runLength = new int[listLength];
+ rleValues = new int[listLength];
+ }
+ }
+
+ private int[] ensureDict(int cardinality) {
+ if (dictKeyList.length < cardinality) {
+ dictKeyList = new int[cardinality];
+ }
+ return dictKeyList;
+ }
+ }
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ private static int smear(int value) {
+ value ^= value >>> 16;
+ value *= 0x7feb352d;
+ value ^= value >>> 15;
+ value *= 0x846ca68b;
+ value ^= value >>> 16;
+ return value;
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ return bitPackingAt(numbers, 0, bitWidth, encodePos, encodedResult, numValues);
+ }
+
+ private static int bitPackingAt(int[] numbers, int offset, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, offset + i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[offset + blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ private static int bitPackingWidth1At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 7) | (numbers[base + 1] << 6)
+ | (numbers[base + 2] << 5) | (numbers[base + 3] << 4)
+ | (numbers[base + 4] << 3) | (numbers[base + 5] << 2)
+ | (numbers[base + 6] << 1) | numbers[base + 7]);
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth2At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 6) | (numbers[base + 1] << 4)
+ | (numbers[base + 2] << 2) | numbers[base + 3]);
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth4At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 4) | numbers[base + 1]);
+ encodePos++;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = encodePos * 8;
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingWidth8At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ for (int i = 0; i < numValues; i++) {
+ encodedResult[encodePos++] = (byte) numbers[offset + i];
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingShifted(int[] list, int listLength, int shiftAmount, int mask,
+ int bitWidth, int encodePos, byte[] encodedResult, int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ fallbackBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+ return bitPacking(fallbackBuffer, bitWidth, encodePos, encodedResult, listLength);
+ }
+
+ private static int bitPackingWidth1Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 8 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 7)
+ | (((list[i + 1] >> shiftAmount) & mask) << 6)
+ | (((list[i + 2] >> shiftAmount) & mask) << 5)
+ | (((list[i + 3] >> shiftAmount) & mask) << 4)
+ | (((list[i + 4] >> shiftAmount) & mask) << 3)
+ | (((list[i + 5] >> shiftAmount) & mask) << 2)
+ | (((list[i + 6] >> shiftAmount) & mask) << 1)
+ | ((list[i + 7] >> shiftAmount) & mask));
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth2Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 4 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 6)
+ | (((list[i + 1] >> shiftAmount) & mask) << 4)
+ | (((list[i + 2] >> shiftAmount) & mask) << 2)
+ | ((list[i + 3] >> shiftAmount) & mask));
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth4Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 2 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 4)
+ | ((list[i + 1] >> shiftAmount) & mask));
+ encodePos++;
+ i += 2;
+ }
+ if (i < listLength) {
+ int bitPos = encodePos * 8;
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingWidth8Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ for (int i = 0; i < listLength; i++) {
+ encodedResult[encodePos++] = (byte) ((list[i] >> shiftAmount) & mask);
+ }
+ return encodePos;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ if (bitWidth == 0) {
+ Arrays.fill(resultList, 0, numValues, 0);
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8(encoded, decodePos, numValues, resultList);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth1(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 7) & 1;
+ resultList[i + 1] = (value >>> 6) & 1;
+ resultList[i + 2] = (value >>> 5) & 1;
+ resultList[i + 3] = (value >>> 4) & 1;
+ resultList[i + 4] = (value >>> 3) & 1;
+ resultList[i + 5] = (value >>> 2) & 1;
+ resultList[i + 6] = (value >>> 1) & 1;
+ resultList[i + 7] = value & 1;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 1);
+ bitPos++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth2(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 6) & 3;
+ resultList[i + 1] = (value >>> 4) & 3;
+ resultList[i + 2] = (value >>> 2) & 3;
+ resultList[i + 3] = value & 3;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 2);
+ bitPos += 2;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth4(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 4) & 15;
+ resultList[i + 1] = value & 15;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ resultList[i] = bytesToInt(encoded, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth8(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ for (int i = 0; i < numValues; i++) {
+ resultList[i] = encoded[decodePos++] & 0xFF;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingOrShifted(
+ byte[] encoded,
+ int decodePos,
+ int bitWidth,
+ int numValues,
+ int[] output,
+ int outputOffset,
+ int shiftAmount,
+ int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+
+ decodePos = decodeBitPacking(encoded, decodePos, bitWidth, numValues, fallbackBuffer);
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= fallbackBuffer[i] << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth1OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 7) & 1) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 6) & 1) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 5) & 1) << shiftAmount;
+ output[outputOffset + i + 3] |= ((value >>> 4) & 1) << shiftAmount;
+ output[outputOffset + i + 4] |= ((value >>> 3) & 1) << shiftAmount;
+ output[outputOffset + i + 5] |= ((value >>> 2) & 1) << shiftAmount;
+ output[outputOffset + i + 6] |= ((value >>> 1) & 1) << shiftAmount;
+ output[outputOffset + i + 7] |= (value & 1) << shiftAmount;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 1) << shiftAmount;
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth2OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 6) & 3) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 4) & 3) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 2) & 3) << shiftAmount;
+ output[outputOffset + i + 3] |= (value & 3) << shiftAmount;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 2) << shiftAmount;
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth4OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 4) & 15) << shiftAmount;
+ output[outputOffset + i + 1] |= (value & 15) << shiftAmount;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 4) << shiftAmount;
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth8OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= (encoded[decodePos++] & 0xFF) << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >>> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >>> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countGroupedRunsUntilLimit(
+ int[] values, int length, int shiftAmount, int mask, int limit) {
+ int previous = (values[0] >>> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >>> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ if (runs >= limit) {
+ return runs;
+ }
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(
+ int[] values,
+ int length,
+ int shiftAmount,
+ int mask,
+ int limit,
+ EncodeScratch scratch) {
+ if (mask == -1 || mask > 31) {
+ scratch.beginDistinctSet(Math.min(limit, length));
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ if (scratch.addDistinctValue((values[i] >>> shiftAmount) & mask)) {
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >>> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ private static int maxDistinctCountToBeat(int currentCost, int valueCount, int beta,
+ int threshold) {
+ int maxDistinct = 0;
+ int maxCandidate = threshold - 1;
+ for (int distinct = 1; distinct <= maxCandidate; distinct++) {
+ int deCost = valueCount * bitWidth(distinct) + distinct * beta;
+ if (deCost < currentCost) {
+ maxDistinct = distinct;
+ }
+ }
+ return maxDistinct;
+ }
+
+ private static int betaLowerBound(
+ int beta,
+ int m,
+ int xLength,
+ int[] bpeCostSingle,
+ int[] rleCostSingle,
+ int[] deCostSingle,
+ Mode mode) {
+ int l = (m + beta - 1) / beta;
+ int lowerBound = 0;
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int best = Integer.MAX_VALUE / 4;
+ if (allowBpe(mode)) {
+ best = Math.min(best, bpeCostSingle[betaStart] * (betaStart - groupStart + 1));
+ }
+
+ int rleCostMax = 0;
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+ if (allowRle(mode)) {
+ best = Math.min(best, rleCostMax);
+ }
+ if (allowDe(mode)) {
+ best = Math.min(best, deCostMax);
+ }
+ lowerBound += best;
+ }
+ return lowerBound;
+ }
+
+ private static int countGroupedRunsAndDistinctUntilLimit(
+ int[] values,
+ int length,
+ int shiftAmount,
+ int mask,
+ int distinctLimit,
+ int[] out,
+ EncodeScratch scratch) {
+ int previous = (values[0] >>> shiftAmount) & mask;
+ int seenMask = 0;
+ if (mask == -1 || mask > 31) {
+ scratch.beginDistinctSet(Math.min(distinctLimit, length));
+ scratch.addDistinctValue(previous);
+ } else {
+ seenMask = 1 << previous;
+ }
+ int runs = 1;
+ int distinctCount = 1;
+
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >>> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ if (mask != -1 && mask <= 31) {
+ int bit = 1 << current;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ }
+ } else if (distinctCount < distinctLimit && scratch.addDistinctValue(current)) {
+ distinctCount++;
+ }
+ }
+
+ out[0] = runs;
+ out[1] = distinctCount >= distinctLimit ? distinctLimit : distinctCount;
+ return distinctCount;
+ }
+
+ private static void extractAllGroups(
+ EncodeScratch scratch,
+ int[] x,
+ int xLength,
+ int beta,
+ int m,
+ int mask) {
+ int l = (m + beta - 1) / beta;
+ int[] flat = scratch.groupFlat;
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ int maxValuePart = 0;
+ int base = i * xLength;
+ for (int j = 0; j < xLength; j++) {
+ int current = (x[j] >>> shiftAmount) & mask;
+ flat[base + j] = current;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ scratch.groupMax[i] = maxValuePart;
+ }
+ scratch.cachedBeta = beta;
+ scratch.cachedL = l;
+ scratch.cachedListLength = xLength;
+ }
+
+ private static boolean useGroupCache(EncodeScratch scratch, int betaValue, int l, int listLength) {
+ return scratch.cachedBeta == betaValue
+ && scratch.cachedL == l
+ && scratch.cachedListLength == listLength;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize, int[] encodingType) {
+ return Subcolumn(x, xLength, m, blockSize, encodingType, Mode.FULL);
+ }
+
+ public static int Subcolumn(
+ int[] x, int xLength, int m, int blockSize, int[] encodingType, Mode mode) {
+ return Subcolumn(x, xLength, m, blockSize, encodingType, ENCODE_SCRATCH.get(), mode);
+ }
+
+ private static int Subcolumn(
+ int[] x,
+ int xLength,
+ int m,
+ int blockSize,
+ int[] encodingType,
+ EncodeScratch scratch,
+ Mode mode) {
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ int[] bpeCostSingle = scratch.bpeCostSingle;
+ int[] rleCostSingle = scratch.rleCostSingle;
+ int[] deCostSingle = scratch.deCostSingle;
+ int[] encodingTypeTemp = scratch.encodingTypeTemp;
+ int[] groupStats = scratch.minDelta3;
+
+ int[] threshold = blockSize == 512 ? THRESHOLD_512 : thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ Arrays.fill(rleCostSingle, 0, m, 1);
+ int valueMask = m == Integer.SIZE ? -1 : (1 << m) - 1;
+ int previousValue = x[0] & valueMask;
+ int unionValue = previousValue;
+ for (int j = 1; j < xLength; j++) {
+ int currentValue = x[j] & valueMask;
+ unionValue |= currentValue;
+ int changedBits = previousValue ^ currentValue;
+ while (changedBits != 0) {
+ int changedBit = Integer.numberOfTrailingZeros(changedBits);
+ rleCostSingle[changedBit]++;
+ changedBits &= changedBits - 1;
+ }
+ previousValue = currentValue;
+ }
+
+ for (int i = 0; i < m; i++) {
+ int runCount = rleCostSingle[i];
+
+ bpeCostSingle[i] = ((unionValue >>> i) & 1) == 1 ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = runCount > 1 ? xLength * 2 + 2 : xLength + 2;
+
+ encodingType[i] = pickSingleBitEncodingType(
+ bpeCostSingle[i], rleCostSingle[i], deCostSingle[i], mode);
+ cost1 += costOfEncodingType(
+ encodingType[i], bpeCostSingle[i], rleCostSingle[i], deCostSingle[i]);
+ }
+
+ int cMin = cost1;
+
+ for (int beta = 2; beta <= m; beta++) {
+ if (betaLowerBound(
+ beta, m, xLength, bpeCostSingle, rleCostSingle, deCostSingle, mode) >= cMin) {
+ continue;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int mask = beta == Integer.SIZE ? -1 : (1 << beta) - 1;
+ int betaThreshold = threshold[beta - 1];
+ for (int t = 0; t < l; t++) {
+ encodingTypeTemp[t] = 0;
+ }
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost;
+ if (allowBpe(mode)) {
+ currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+ } else {
+ currentCost = Integer.MAX_VALUE / 4;
+ }
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ boolean needRle = allowRle(mode) && rleCostMax < currentCost;
+
+ if (allowRle(mode) && needRle) {
+ int rleUnitCost = beta + lengthBitWidth;
+ int maxRunsToBeat = (currentCost - 1) / rleUnitCost;
+ int runCount = countGroupedRunsUntilLimit(
+ x, xLength, groupStart, mask, maxRunsToBeat + 1);
+ if (runCount <= maxRunsToBeat) {
+ int rleCost = runCount * rleUnitCost;
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ if (allowDe(mode) && deCostMax < currentCost) {
+ int maxDistinctToBeat =
+ maxDistinctCountToBeat(currentCost, xLength, beta, betaThreshold);
+ if (maxDistinctToBeat > 0) {
+ int distinctLimit = Math.min(betaThreshold, maxDistinctToBeat + 1);
+ int distinctCount = countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, distinctLimit, scratch);
+ if (distinctCount <= maxDistinctToBeat) {
+ int deCost =
+ xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+ }
+
+ cost += currentCost;
+ if (cost >= cMin) {
+ break;
+ }
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ if (betaBest > 1) {
+ int bestMask = betaBest == Integer.SIZE ? -1 : (1 << betaBest) - 1;
+ extractAllGroups(scratch, x, xLength, betaBest, m, bestMask);
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType) {
+ return SubcolumnEncoder(list, list.length, encodePos, encodedResult, beta, blockSize,
+ encodingType, -1);
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType, int knownM) {
+ return SubcolumnEncoder(list, list.length, encodePos, encodedResult, beta, blockSize,
+ encodingType, knownM);
+ }
+
+ public static int SubcolumnEncoder(
+ int[] list,
+ int listLength,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ int blockSize,
+ int[] encodingType,
+ int knownM) {
+ int m = knownM;
+ if (m < 0) {
+ int maxValue = 0;
+ for (int i = 0; i < listLength; i++) {
+ int value = list[i];
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+ m = bitWidth(maxValue);
+ }
+ return SubcolumnEncoder(list, listLength, encodePos, encodedResult, beta, blockSize,
+ encodingType, m, ENCODE_SCRATCH.get());
+ }
+
+ private static int encodeRleRuns(
+ int[] runLength,
+ int[] rleValues,
+ int runCount,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(runLength, runLengthBitWidth, encodePos, encodedResult, runCount);
+ return bitPacking(rleValues, valueBitWidth, encodePos, encodedResult, runCount);
+ }
+
+ private static int encodeRleFromValues(
+ int[] values,
+ int offset,
+ int listLength,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = values[offset];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = values[offset + j];
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ private static int encodeRleShifted(
+ int[] list,
+ int listLength,
+ int shiftAmount,
+ int mask,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = (list[0] >> shiftAmount) & mask;
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ public static int[] borrowMinDelta3Buffer() {
+ return ENCODE_SCRATCH.get().minDelta3;
+ }
+
+ public static int[] borrowEncodingTypeBuffer() {
+ return ENCODE_SCRATCH.get().encodingType;
+ }
+
+ public static int[] borrowDataDeltaBuffer() {
+ return ENCODE_SCRATCH.get().dataDelta;
+ }
+
+ private static int SubcolumnEncoder(
+ int[] list,
+ int listLength,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ int blockSize,
+ int[] encodingType,
+ int m,
+ EncodeScratch scratch) {
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = scratch.bitWidthList;
+ int[] subcolumnBuffer = scratch.subcolumnBuffer;
+ int[] runLength = scratch.runLength;
+ int[] rleValues = scratch.rleValues;
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = betaValue == Integer.SIZE ? -1 : (1 << betaValue) - 1;
+ boolean useCache = useGroupCache(scratch, betaValue, l, listLength);
+
+ if (useCache) {
+ for (int i = 0; i < l; i++) {
+ bitWidthList[i] = bitWidth(scratch.groupMax[i]);
+ }
+ } else if (betaValue == 1) {
+ int unionValue = 0;
+ for (int j = 0; j < listLength; j++) {
+ unionValue |= list[j];
+ }
+ for (int i = 0; i < l; i++) {
+ bitWidthList[i] = (unionValue >>> i) & 1;
+ }
+ } else {
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ int maxValuePart = 0;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ int groupOffset = i * listLength;
+
+ if (encodingType[i] == 0) {
+ if (useCache) {
+ encodePos = bitPackingAt(scratch.groupFlat, groupOffset, bitWidthList[i],
+ encodePos, encodedResult, listLength);
+ } else {
+ encodePos = bitPackingShifted(list, listLength, shiftAmount, mask,
+ bitWidthList[i], encodePos, encodedResult, subcolumnBuffer);
+ }
+ continue;
+ }
+
+ if (encodingType[i] == 1) {
+ if (useCache) {
+ encodePos = encodeRleFromValues(scratch.groupFlat, groupOffset, listLength,
+ runLength, rleValues, bw, bitWidthList[i], encodePos, encodedResult);
+ } else {
+ encodePos = encodeRleShifted(list, listLength, shiftAmount, mask, runLength,
+ rleValues, bw, bitWidthList[i], encodePos, encodedResult);
+ }
+ continue;
+ }
+
+ boolean compactDictionary = mask != -1 && mask <= 31;
+ int cardinality;
+ int[] dictKeyList;
+
+ if (compactDictionary) {
+ int seenMask = 0;
+ if (useCache) {
+ System.arraycopy(scratch.groupFlat, groupOffset, subcolumnBuffer, 0,
+ listLength);
+ for (int j = 0; j < listLength; j++) {
+ seenMask |= 1 << subcolumnBuffer[j];
+ }
+ } else {
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >>> shiftAmount) & mask;
+ subcolumnBuffer[j] = current;
+ seenMask |= 1 << current;
+ }
+ }
+
+ cardinality = Integer.bitCount(seenMask);
+ dictKeyList = scratch.ensureDictKeyList(cardinality);
+ int[] codeMap = scratch.ensureCodeMap(mask + 1);
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if ((seenMask & (1 << value)) != 0) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+ } else {
+ dictKeyList = scratch.ensureDictKeyList(listLength);
+ HashMap<Integer, Integer> dictCodeMap = scratch.dictCodeMap;
+ dictCodeMap.clear();
+ int dictSize = 0;
+ if (useCache) {
+ System.arraycopy(scratch.groupFlat, groupOffset, subcolumnBuffer, 0,
+ listLength);
+ } else {
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = (list[j] >>> shiftAmount) & mask;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ int value = subcolumnBuffer[j];
+ Integer code = dictCodeMap.get(value);
+ if (code == null) {
+ code = dictSize;
+ dictCodeMap.put(value, code);
+ dictKeyList[dictSize] = value;
+ dictSize++;
+ }
+ subcolumnBuffer[j] = code;
+ }
+ cardinality = dictSize;
+ }
+
+ int dictBitWidth = bitWidth(cardinality);
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ return SubcolumnDecoder(encodedResult, encodePos, list, 0, list.length, blockSize);
+ }
+
+ private static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int outputOffset, int listLength, int blockSize) {
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ DecodeScratch scratch = DECODE_SCRATCH.get();
+ scratch.ensureL(l);
+ scratch.ensureListLength(listLength);
+
+ int[] bitWidthList = scratch.bitWidthList;
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = scratch.encodingType;
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = scratch.subcolumnBuffer;
+ int[] runLength = scratch.runLength;
+ int[] rleValues = scratch.rleValues;
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+ int shiftAmount = i * beta;
+
+ if (type == 0) {
+ encodePos = decodeBitPackingOrShifted(encodedResult, encodePos, currentBitWidth,
+ listLength, list, outputOffset, shiftAmount, subcolumnBuffer);
+ continue;
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j] << shiftAmount;
+ int outputBase = outputOffset + currentIndex;
+ for (int k = currentIndex; k < endPos; k++) {
+ list[outputBase++] |= value;
+ }
+ currentIndex = endPos;
+ }
+ continue;
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = scratch.ensureDict(cardinality);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ list[outputOffset + j] |= dictKeyList[subcolumnBuffer[j]] << shiftAmount;
+ }
+ continue;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] out = new int[remaining];
+ fillAbsDeltaTsBlock(tsBlock, blockIndex, blockSize, remaining, minDelta, out);
+ return out;
+ }
+
+ private static int fillAbsDeltaTsBlock(
+ int[] tsBlock,
+ int blockIndex,
+ int blockSize,
+ int remaining,
+ int[] minDelta,
+ int[] out) {
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int valueDeltaMax = Integer.MIN_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ if (current > valueDeltaMax) {
+ valueDeltaMax = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ out[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return valueDeltaMax - valueDeltaMin;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ return BlockEncoder(data, blockIndex, blockSize, remainder, encodePos, encodedResult, beta,
+ null, null, Mode.FULL);
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta, long[] forTime,
+ long[] subcolumnTime) {
+ return BlockEncoder(data, blockIndex, blockSize, remainder, encodePos, encodedResult, beta,
+ forTime, subcolumnTime, Mode.FULL);
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta, long[] forTime,
+ long[] subcolumnTime, Mode mode) {
+ EncodeScratch scratch = ENCODE_SCRATCH.get();
+ long forStart = System.nanoTime();
+ int maxValue = fillAbsDeltaTsBlock(
+ data, blockIndex, blockSize, remainder, scratch.minDelta, scratch.dataDelta);
+ long forEnd = System.nanoTime();
+ if (forTime != null) {
+ forTime[0] += (forEnd - forStart);
+ }
+
+ long subStart = System.nanoTime();
+ int2Bytes(scratch.minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int m = bitWidth(maxValue);
+ beta[0] = Subcolumn(scratch.dataDelta, remainder, m, blockSize, scratch.encodingType, scratch, mode);
+ encodePos = SubcolumnEncoder(scratch.dataDelta, remainder, encodePos, encodedResult, beta,
+ blockSize, scratch.encodingType, m, scratch);
+ long subEnd = System.nanoTime();
+ if (subcolumnTime != null) {
+ subcolumnTime[0] += (subEnd - subStart);
+ }
+ return encodePos;
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int base = blockIndex * blockSize;
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, data, base, remainder, blockSize);
+
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] += minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, Mode.FULL);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, Mode mode) {
+ return Encoder(data, blockSize, encodedResult, null, null, mode);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, long[] forTime,
+ long[] subcolumnTime) {
+ return Encoder(data, blockSize, encodedResult, forTime, subcolumnTime, Mode.FULL);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, long[] forTime,
+ long[] subcolumnTime, Mode mode) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = ENCODE_SCRATCH.get().beta;
+ beta[0] = 2;
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult,
+ beta, forTime, subcolumnTime, mode);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockEncoder(data, numBlocks, blockSize, remainder, encodePos,
+ encodedResult, beta, forTime, subcolumnTime, mode);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static void runAblationBenchmark(String outputPath, Mode mode) throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+
+ int blockSize = 512;
+ int warmupTime = 20;
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, data2Arr.length * 8)];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double compressedSize = 0;
+ int length = 0;
+
+ for (int repeat = 0; repeat < warmupTime; repeat++) {
+ length = Encoder(data2Arr, blockSize, encodedResult, mode);
+ }
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2Arr, blockSize, encodedResult, mode);
+ }
+ long e = System.nanoTime();
+ encodeTime += (e - s) / repeatTime;
+ compressedSize += length;
+
+ double ratio = compressedSize / (double) (data1.size() * Long.BYTES);
+
+ for (int repeat = 0; repeat < warmupTime; repeat++) {
+ Decoder(encodedResult);
+ }
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s) / repeatTime;
+
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio)
+ });
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateAppendTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateAppendTest.java
new file mode 100644
index 0000000..7b206c0
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateAppendTest.java
@@ -0,0 +1,118 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateAppendTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_append.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Append-only Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+
+ int appendValue = (stats.maxValue == Integer.MAX_VALUE) ? stats.maxValue : (stats.maxValue + 1);
+
+ int[] appended = new int[origin.length + 1];
+ System.arraycopy(origin, 0, appended, 0, origin.length);
+ appended[origin.length] = appendValue;
+
+ byte[] encodedResult = new byte[Math.max(16, appended.length * 12)];
+ int encodedLength;
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+
+ long appendCompressTime;
+ int appendedLength = encodedLength;
+ int[] beta = new int[] {2};
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int remainder = tailInfo.remainder;
+ int numBlocks = tailInfo.numBlocks;
+ int tailStart = tailInfo.tailStartPos;
+ int newRemainder = remainder + 1;
+
+ int encodePos = tailStart;
+ if (newRemainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(appended[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ appended, numBlocks, blockSize, newRemainder, encodePos, encodedResult, beta);
+ }
+ appendedLength = encodePos;
+ }
+ end = System.nanoTime();
+ appendCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(appendCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(appendedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateDeleteSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateDeleteSmallerTest.java
new file mode 100644
index 0000000..c4a7e76
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateDeleteSmallerTest.java
@@ -0,0 +1,124 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateDeleteSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_delete_smaller.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Delete Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length <= 1) {
+ continue;
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ long start = System.nanoTime();
+ int encodedLength = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+ if (tailInfo.remainder <= 0) {
+ continue;
+ }
+
+ int[] deleted = new int[origin.length - 1];
+ System.arraycopy(origin, 0, deleted, 0, deleted.length);
+
+ long deleteCompressTime;
+ int updatedLength = encodedLength;
+ int[] beta = new int[] {2};
+ int newRemainder = tailInfo.remainder - 1;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ if (newRemainder == 0) {
+ updatedLength = tailInfo.tailStartPos;
+ continue;
+ }
+ int encodePos = tailInfo.tailStartPos;
+ if (newRemainder <= 3) {
+ int base = tailInfo.numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(deleted[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ deleted,
+ tailInfo.numBlocks,
+ blockSize,
+ newRemainder,
+ encodePos,
+ encodedResult,
+ beta);
+ }
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ deleteCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(deleteCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertLargerTest.java
new file mode 100644
index 0000000..c24c864
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertLargerTest.java
@@ -0,0 +1,122 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateInsertLargerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_insert_larger.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length == 0) {
+ continue;
+ }
+
+ int insertValue = (stats.maxValue == Integer.MAX_VALUE) ? stats.maxValue : stats.maxValue + 1;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = insertValue;
+
+ byte[] encodedResult = new byte[Math.max(16, inserted.length * 12)];
+ long start = System.nanoTime();
+ int encodedLength = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+ if (tailInfo.remainder <= 0) {
+ continue;
+ }
+
+ long insertCompressTime;
+ int updatedLength = encodedLength;
+ int[] beta = new int[] {2};
+ int newRemainder = tailInfo.remainder + 1;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos = tailInfo.tailStartPos;
+ if (newRemainder <= 3) {
+ int base = tailInfo.numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(inserted[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ inserted,
+ tailInfo.numBlocks,
+ blockSize,
+ newRemainder,
+ encodePos,
+ encodedResult,
+ beta);
+ }
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ insertCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(insertCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertSmallerTest.java
new file mode 100644
index 0000000..62f1376
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateInsertSmallerTest.java
@@ -0,0 +1,122 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateInsertSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_insert_smaller.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length == 0) {
+ continue;
+ }
+
+ int insertValue = (stats.minValue == Integer.MIN_VALUE) ? stats.minValue : stats.minValue - 1;
+ int[] inserted = new int[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = insertValue;
+
+ byte[] encodedResult = new byte[Math.max(16, inserted.length * 12)];
+ long start = System.nanoTime();
+ int encodedLength = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+ if (tailInfo.remainder <= 0) {
+ continue;
+ }
+
+ long insertCompressTime;
+ int updatedLength = encodedLength;
+ int[] beta = new int[] {2};
+ int newRemainder = tailInfo.remainder + 1;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos = tailInfo.tailStartPos;
+ if (newRemainder <= 3) {
+ int base = tailInfo.numBlocks * blockSize;
+ for (int i = 0; i < newRemainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(inserted[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ inserted,
+ tailInfo.numBlocks,
+ blockSize,
+ newRemainder,
+ encodePos,
+ encodedResult,
+ beta);
+ }
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ insertCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(insertCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateLargerTest.java
new file mode 100644
index 0000000..eb38003
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateLargerTest.java
@@ -0,0 +1,122 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateLargerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_larger.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length == 0) {
+ continue;
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ long start = System.nanoTime();
+ int encodedLength = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+ if (tailInfo.remainder <= 0) {
+ continue;
+ }
+
+ int updateIndex = tailInfo.numBlocks * blockSize + tailInfo.remainder - 1;
+ int updatedValue = (stats.maxValue == Integer.MAX_VALUE) ? stats.maxValue : stats.maxValue + 1;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[updateIndex] = updatedValue;
+
+ long updateCompressTime;
+ int updatedLength = encodedLength;
+ int[] beta = new int[] {2};
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos = tailInfo.tailStartPos;
+ if (tailInfo.remainder <= 3) {
+ int base = tailInfo.numBlocks * blockSize;
+ for (int i = 0; i < tailInfo.remainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(updated[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ updated,
+ tailInfo.numBlocks,
+ blockSize,
+ tailInfo.remainder,
+ encodePos,
+ encodedResult,
+ beta);
+ }
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ updateCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(updateCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateRatioTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateRatioTest.java
new file mode 100644
index 0000000..7d09a71
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateRatioTest.java
@@ -0,0 +1,656 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/**
+ * 按数据点比例（0%～100%）选取更新点，比较两种块级更新策略：
+ *
+ * <p>1) 固定策略：不重算最优 beta / 编码类型，复用原块 beta 与原编码类型模板重编码；
+ *
+ * <p>2) 自适应策略：重算最优 beta / 编码类型（调用 BlockEncoder）。
+ */
+public class SubcolumnAddDictPruneNewUpdateRatioTest {
+
+ // private static final int[] UPDATE_RATIOS = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100};
+ private static final int[] UPDATE_RATIOS = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20};
+
+ /**
+ * If true, updated indices form one contiguous window (centered). At low ratios only a few
+ * blocks are dirty, so compression-vs-update-ratio curves (especially re-optimizing α) change
+ * gradually from 0% to 1%. If false, legacy uniform spread: ~1% point updates still touch almost
+ * every block, which produces a sharp 0–1% drop in plots.
+ */
+ private static final boolean CLUSTER_UPDATES_IN_CONTIGUOUS_WINDOW = true;
+
+ private static final int STATS_UPDATED_LENGTH = 0;
+ private static final int STATS_BETA_CHANGED = 1;
+ private static final int STATS_REENCODED_BLOCKS = 2;
+ private static final int STATS_MEMCPY_BLOCKS = 3;
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_ratio.csv";
+
+ int blockSize = 512;
+ int repeatTime = 1000;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Update Ratio(%)",
+ "Total Blocks",
+ "Affected Blocks",
+ "Updated Points",
+ "Encoding Time(ns)",
+ "Update Time Fixed(ns)",
+ "Compressed Size(Fixed)",
+ "Compression Ratio(Fixed)",
+ "Update Time Recompute(ns)",
+ "Compressed Size(Recompute)",
+ "Compression Ratio(Recompute)",
+ "Need Recompute Beta",
+ "Beta Changed Blocks",
+ "Memcpy Blocks(Fixed)",
+ "Reencoded Blocks(Fixed)",
+ "Memcpy Blocks(Recompute)",
+ "Reencoded Blocks(Recompute)",
+ "Points",
+ "Compressed Size(Original)",
+ "Compression Ratio(Original)"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length == 0) {
+ continue;
+ }
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+
+ int[] metaStart = new int[totalBlocks];
+ int[] metaEnd = new int[totalBlocks];
+ int[] metaRowCount = new int[totalBlocks];
+ int[] metaBeta = new int[totalBlocks];
+ boolean[] metaSubcolumn = new boolean[totalBlocks];
+ int[][] metaEncodingType = new int[totalBlocks][];
+
+ byte[] encodedOrigin = new byte[Math.max(16, origin.length * 20)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedOrigin);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedOrigin);
+
+ parseBlockMetas(
+ encodedOrigin,
+ encodedLength,
+ blockSize,
+ metaStart,
+ metaEnd,
+ metaRowCount,
+ metaBeta,
+ metaSubcolumn,
+ metaEncodingType);
+
+ int[] fixedStats = new int[4];
+ int[] recomputeStats = new int[4];
+
+ for (int ratio : UPDATE_RATIOS) {
+ int[] updated = buildUpdatedValues(origin, ratio, blockSize);
+ boolean[] affected = markAffectedBlocks(origin, updated, blockSize, totalBlocks);
+ int updatedPoints = countUpdatedPoints(origin, updated);
+ int affectedBlockCount = countTrue(affected);
+
+ byte[] encodedUpdatedFixed = new byte[Math.max(16, origin.length * 20)];
+ byte[] encodedUpdatedRecompute = new byte[Math.max(16, origin.length * 20)];
+
+ long updateStart = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ mergeEncodedByBlockFixed(
+ updated,
+ blockSize,
+ numBlocks,
+ remainder,
+ encodedOrigin,
+ metaStart,
+ metaEnd,
+ metaRowCount,
+ metaBeta,
+ metaSubcolumn,
+ metaEncodingType,
+ affected,
+ encodedUpdatedFixed,
+ fixedStats);
+ }
+ long updateEnd = System.nanoTime();
+ long updateTimeFixed = (updateEnd - updateStart) / repeatTime;
+
+ updateStart = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ mergeEncodedByBlockRecompute(
+ updated,
+ blockSize,
+ numBlocks,
+ remainder,
+ encodedOrigin,
+ metaStart,
+ metaEnd,
+ metaRowCount,
+ metaBeta,
+ metaSubcolumn,
+ affected,
+ encodedUpdatedRecompute,
+ recomputeStats);
+ }
+ updateEnd = System.nanoTime();
+ long updateTimeRecompute = (updateEnd - updateStart) / repeatTime;
+
+ int updatedLengthFixed = fixedStats[STATS_UPDATED_LENGTH];
+ int updatedLengthRecompute = recomputeStats[STATS_UPDATED_LENGTH];
+ int betaChangedBlocks = recomputeStats[STATS_BETA_CHANGED];
+ int fixedReencodedBlocks = fixedStats[STATS_REENCODED_BLOCKS];
+ int fixedMemcpyBlocks = fixedStats[STATS_MEMCPY_BLOCKS];
+ int recomputeReencodedBlocks = recomputeStats[STATS_REENCODED_BLOCKS];
+ int recomputeMemcpyBlocks = recomputeStats[STATS_MEMCPY_BLOCKS];
+
+ double originalRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ double fixedRatio = updatedLengthFixed / (double) (origin.length * Long.BYTES);
+ double recomputeRatio = updatedLengthRecompute / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(ratio),
+ String.valueOf(totalBlocks),
+ String.valueOf(affectedBlockCount),
+ String.valueOf(updatedPoints),
+ String.valueOf(encodeTime),
+ String.valueOf(updateTimeFixed),
+ String.valueOf(updatedLengthFixed),
+ String.valueOf(fixedRatio),
+ String.valueOf(updateTimeRecompute),
+ String.valueOf(updatedLengthRecompute),
+ String.valueOf(recomputeRatio),
+ String.valueOf(betaChangedBlocks > 0),
+ String.valueOf(betaChangedBlocks),
+ String.valueOf(fixedMemcpyBlocks),
+ String.valueOf(fixedReencodedBlocks),
+ String.valueOf(recomputeMemcpyBlocks),
+ String.valueOf(recomputeReencodedBlocks),
+ String.valueOf(origin.length),
+ String.valueOf(encodedLength),
+ String.valueOf(originalRatio)
+ });
+ }
+ }
+ writer.close();
+ }
+
+ private static int[] buildUpdatedValues(int[] origin, int ratio, int blockSize) {
+ int[] updated = Arrays.copyOf(origin, origin.length);
+ if (origin.length == 0) {
+ return updated;
+ }
+
+ int targetUpdateCount = (int) Math.ceil(origin.length * (ratio / 100.0));
+ if (targetUpdateCount <= 0) {
+ return updated;
+ }
+
+ if (CLUSTER_UPDATES_IN_CONTIGUOUS_WINDOW) {
+ int window = Math.min(targetUpdateCount, origin.length);
+ int start = Math.max(0, (origin.length - window) / 2);
+ for (int i = 0; i < window; i++) {
+ int index = start + i;
+ int prevValue = (index > 0) ? origin[index - 1] : origin[index];
+ int nextValue = (index < origin.length - 1) ? origin[index + 1] : origin[index];
+ updated[index] = (prevValue + nextValue) / 2;
+ }
+ return updated;
+ }
+
+ int blockCount = (origin.length + blockSize - 1) / blockSize;
+ int[] blockMin = new int[blockCount];
+ int[] blockMax = new int[blockCount];
+ Arrays.fill(blockMin, Integer.MAX_VALUE);
+ Arrays.fill(blockMax, Integer.MIN_VALUE);
+ for (int i = 0; i < origin.length; i++) {
+ int blockIndex = i / blockSize;
+ int value = origin[i];
+ if (value < blockMin[blockIndex]) {
+ blockMin[blockIndex] = value;
+ }
+ if (value > blockMax[blockIndex]) {
+ blockMax[blockIndex] = value;
+ }
+ }
+
+ // Random random = new Random();
+ boolean[] used = new boolean[origin.length];
+ for (int i = 0; i < targetUpdateCount; i++) {
+ int index = (int) (((long) i * origin.length) / targetUpdateCount);
+ if (index >= origin.length) {
+ index = origin.length - 1;
+ }
+ while (used[index]) {
+ index++;
+ if (index >= origin.length) {
+ index = 0;
+ }
+ }
+ used[index] = true;
+
+ int prevValue = (index > 0) ? origin[index - 1] : origin[index];
+ int nextValue = (index < origin.length - 1) ? origin[index + 1] : origin[index];
+ updated[index] = (prevValue + nextValue) / 2;
+ // updated[index] = nextValue;
+
+ /*
+ int blockIndex = index / blockSize;
+ int minValue = blockMin[blockIndex];
+ int maxValue = blockMax[blockIndex];
+ if (minValue >= maxValue) {
+ updated[index] = maxValue;
+ continue;
+ }
+
+ int oldValue = origin[index];
+ int candidate = oldValue;
+ for (int attempt = 0; attempt < 6 && candidate == oldValue; attempt++) {
+ long span = (long) maxValue - (long) minValue + 1L;
+ long offset = nextLongBounded(random, span);
+ candidate = (int) ((long) minValue + offset);
+ }
+ if (candidate == oldValue) {
+ candidate = (oldValue == maxValue) ? oldValue - 1 : oldValue + 1;
+ }
+ updated[index] = candidate;
+ */
+ }
+ return updated;
+ }
+
+ /*
+ private static long nextLongBounded(Random random, long bound) {
+ if (bound <= 0L) {
+ throw new IllegalArgumentException("bound must be positive");
+ }
+ long m = bound - 1L;
+ if ((bound & m) == 0L) {
+ return random.nextLong() & m;
+ }
+ long u = random.nextLong() >>> 1;
+ while (u + m - (u % bound) < 0L) {
+ u = random.nextLong() >>> 1;
+ }
+ return u % bound;
+ }
+ */
+
+ private static boolean[] markAffectedBlocks(
+ int[] origin, int[] updated, int blockSize, int blockCount) {
+ boolean[] affected = new boolean[blockCount];
+ for (int i = 0; i < origin.length; i++) {
+ if (origin[i] != updated[i]) {
+ int blockIndex = i / blockSize;
+ if (blockIndex >= 0 && blockIndex < affected.length) {
+ affected[blockIndex] = true;
+ }
+ }
+ }
+ return affected;
+ }
+
+ private static int countTrue(boolean[] flags) {
+ int c = 0;
+ for (boolean f : flags) {
+ if (f) {
+ c++;
+ }
+ }
+ return c;
+ }
+
+ private static int countUpdatedPoints(int[] origin, int[] updated) {
+ int changed = 0;
+ for (int i = 0; i < origin.length; i++) {
+ if (origin[i] != updated[i]) {
+ changed++;
+ }
+ }
+ return changed;
+ }
+
+ /** 固定策略：复用原块 beta 与原编码类型模板，不重算最优 beta/编码类型。 */
+ private static void mergeEncodedByBlockFixed(
+ int[] updated,
+ int blockSize,
+ int numBlocks,
+ int remainder,
+ byte[] encodedOrigin,
+ int[] metaStart,
+ int[] metaEnd,
+ int[] metaRowCount,
+ int[] metaBeta,
+ boolean[] metaSubcolumn,
+ int[][] metaEncodingType,
+ boolean[] affected,
+ byte[] outputBuffer,
+ int[] mergeStats) {
+ SubcolumnPruneNewTest.int2Bytes(updated.length, 0, outputBuffer);
+ SubcolumnPruneNewTest.int2Bytes(blockSize, 4, outputBuffer);
+
+ int encodePos = 8;
+ int reencodedBlocks = 0;
+ int memcpyBlocks = 0;
+ int totalBlocks = metaStart.length;
+
+ for (int i = 0; i < totalBlocks; i++) {
+ if (!affected[i]) {
+ int len = metaEnd[i] - metaStart[i];
+ System.arraycopy(encodedOrigin, metaStart[i], outputBuffer, encodePos, len);
+ encodePos += len;
+ memcpyBlocks++;
+ continue;
+ }
+
+ reencodedBlocks++;
+ if (!metaSubcolumn[i]) {
+ int encIdx = encoderBlockIndex(i, numBlocks, remainder);
+ int base = encIdx * blockSize;
+ for (int k = 0; k < metaRowCount[i]; k++) {
+ SubcolumnPruneNewTest.int2Bytes(updated[base + k], encodePos, outputBuffer);
+ encodePos += 4;
+ }
+ continue;
+ }
+
+ int encIdx = encoderBlockIndex(i, numBlocks, remainder);
+ int fixedBeta = metaBeta[i] > 0 ? metaBeta[i] : 2;
+ encodePos =
+ blockEncodeWithFixedScheme(
+ updated,
+ encIdx,
+ blockSize,
+ metaRowCount[i],
+ encodePos,
+ outputBuffer,
+ fixedBeta,
+ metaEncodingType[i]);
+ }
+
+ mergeStats[STATS_UPDATED_LENGTH] = encodePos;
+ mergeStats[STATS_BETA_CHANGED] = 0;
+ mergeStats[STATS_REENCODED_BLOCKS] = reencodedBlocks;
+ mergeStats[STATS_MEMCPY_BLOCKS] = memcpyBlocks;
+ }
+
+ /** 自适应策略：重算每个更新块的最优 beta/编码类型。 */
+ private static void mergeEncodedByBlockRecompute(
+ int[] updated,
+ int blockSize,
+ int numBlocks,
+ int remainder,
+ byte[] encodedOrigin,
+ int[] metaStart,
+ int[] metaEnd,
+ int[] metaRowCount,
+ int[] metaBeta,
+ boolean[] metaSubcolumn,
+ boolean[] affected,
+ byte[] outputBuffer,
+ int[] mergeStats) {
+ SubcolumnPruneNewTest.int2Bytes(updated.length, 0, outputBuffer);
+ SubcolumnPruneNewTest.int2Bytes(blockSize, 4, outputBuffer);
+
+ int encodePos = 8;
+ int betaChangedBlocks = 0;
+ int reencodedBlocks = 0;
+ int memcpyBlocks = 0;
+ int totalBlocks = metaStart.length;
+
+ for (int i = 0; i < totalBlocks; i++) {
+ if (!affected[i]) {
+ int len = metaEnd[i] - metaStart[i];
+ System.arraycopy(encodedOrigin, metaStart[i], outputBuffer, encodePos, len);
+ encodePos += len;
+ memcpyBlocks++;
+ continue;
+ }
+
+ reencodedBlocks++;
+ if (!metaSubcolumn[i]) {
+ int encIdx = encoderBlockIndex(i, numBlocks, remainder);
+ int base = encIdx * blockSize;
+ for (int k = 0; k < metaRowCount[i]; k++) {
+ SubcolumnPruneNewTest.int2Bytes(updated[base + k], encodePos, outputBuffer);
+ encodePos += 4;
+ }
+ continue;
+ }
+
+ int[] beta = new int[] {2};
+ int encIdx = encoderBlockIndex(i, numBlocks, remainder);
+ int newPos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ updated, encIdx, blockSize, metaRowCount[i], encodePos, outputBuffer, beta);
+ if (metaBeta[i] > 0 && beta[0] != metaBeta[i]) {
+ betaChangedBlocks++;
+ }
+ encodePos = newPos;
+ }
+
+ mergeStats[STATS_UPDATED_LENGTH] = encodePos;
+ mergeStats[STATS_BETA_CHANGED] = betaChangedBlocks;
+ mergeStats[STATS_REENCODED_BLOCKS] = reencodedBlocks;
+ mergeStats[STATS_MEMCPY_BLOCKS] = memcpyBlocks;
+ }
+
+ private static int blockEncodeWithFixedScheme(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int rowCount,
+ int encodePos,
+ byte[] encodedResult,
+ int fixedBeta,
+ int[] originalEncodingType) {
+ int[] minDelta = new int[1];
+ int[] dataDelta =
+ SubcolumnPruneNewTest.getAbsDeltaTsBlock(data, blockIndex, blockSize, rowCount, minDelta);
+
+ SubcolumnPruneNewTest.int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < rowCount; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ int m = SubcolumnPruneNewTest.bitWidth(maxValue);
+ int betaValue = fixedBeta > 0 ? fixedBeta : 2;
+ int[] beta = new int[] {betaValue};
+ int[] encodingType = adaptEncodingType(originalEncodingType, m, betaValue);
+ return SubcolumnPruneNewTest.SubcolumnEncoder(
+ dataDelta, encodePos, encodedResult, beta, blockSize, encodingType);
+ }
+
+ private static int[] adaptEncodingType(int[] originalEncodingType, int m, int betaValue) {
+ if (m == 0) {
+ return new int[0];
+ }
+ int l = (m + betaValue - 1) / betaValue;
+ int[] result = new int[l];
+ if (originalEncodingType == null) {
+ return result;
+ }
+ int copyLen = Math.min(l, originalEncodingType.length);
+ for (int i = 0; i < copyLen; i++) {
+ int type = originalEncodingType[i];
+ result[i] = (type >= 0 && type <= 2) ? type : 0;
+ }
+ return result;
+ }
+
+ private static int encoderBlockIndex(int blockSlot, int numBlocks, int remainder) {
+ if (remainder > 0 && blockSlot == numBlocks) {
+ return numBlocks;
+ }
+ return blockSlot;
+ }
+
+ private static void parseBlockMetas(
+ byte[] encoded,
+ int encodedLength,
+ int expectedBlockSize,
+ int[] metaStart,
+ int[] metaEnd,
+ int[] metaRowCount,
+ int[] metaBeta,
+ boolean[] metaSubcolumn,
+ int[][] metaEncodingType) {
+ int dataLength = SubcolumnPruneNewTest.bytes2Integer(encoded, 0, 4);
+ int parsedBlockSize = SubcolumnPruneNewTest.bytes2Integer(encoded, 4, 4);
+ if (parsedBlockSize != expectedBlockSize) {
+ throw new IllegalStateException("blockSize mismatch header vs test.");
+ }
+ int numBlocks = dataLength / parsedBlockSize;
+ int remainder = dataLength % parsedBlockSize;
+ int pos = 8;
+ int tb = 0;
+
+ for (int blockIndex = 0; blockIndex < numBlocks; blockIndex++) {
+ metaStart[tb] = pos;
+ metaRowCount[tb] = parsedBlockSize;
+ int[] betaHolder = new int[1];
+ pos =
+ parseSubcolumnBlock(
+ encoded,
+ pos,
+ parsedBlockSize,
+ parsedBlockSize,
+ betaHolder,
+ metaEncodingType,
+ tb);
+ metaBeta[tb] = betaHolder[0];
+ metaSubcolumn[tb] = true;
+ metaEnd[tb] = pos;
+ tb++;
+ }
+
+ if (remainder > 0) {
+ metaStart[tb] = pos;
+ metaRowCount[tb] = remainder;
+ if (remainder <= 3) {
+ metaSubcolumn[tb] = false;
+ metaBeta[tb] = -1;
+ metaEncodingType[tb] = new int[0];
+ pos += remainder * 4;
+ } else {
+ int[] betaHolder = new int[1];
+ pos =
+ parseSubcolumnBlock(
+ encoded, pos, parsedBlockSize, remainder, betaHolder, metaEncodingType, tb);
+ metaBeta[tb] = betaHolder[0];
+ metaSubcolumn[tb] = true;
+ }
+ metaEnd[tb] = pos;
+ tb++;
+ }
+
+ if (pos > encodedLength) {
+ throw new IllegalStateException("Encoded stream parsing overflow.");
+ }
+ }
+
+ private static int parseSubcolumnBlock(
+ byte[] encoded,
+ int encodePos,
+ int blockSize,
+ int rowCount,
+ int[] betaOut,
+ int[][] metaEncodingType,
+ int blockSlot) {
+ encodePos += 4;
+ int m = SubcolumnPruneNewTest.bytes2Integer(encoded, encodePos, 1);
+ encodePos += 1;
+ if (m == 0) {
+ betaOut[0] = 1;
+ metaEncodingType[blockSlot] = new int[0];
+ return encodePos;
+ }
+
+ int beta = SubcolumnPruneNewTest.bytes2Integer(encoded, encodePos, 1);
+ betaOut[0] = beta;
+ encodePos += 1;
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnPruneNewTest.decodeBitPacking(encoded, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnPruneNewTest.decodeBitPacking(encoded, encodePos, 2, l, encodingType);
+ metaEncodingType[blockSlot] = Arrays.copyOf(encodingType, l);
+
+ int bw = SubcolumnPruneNewTest.bitWidth(blockSize);
+ int scanPos = encodePos;
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ long bitPos = ((long) scanPos) * 8L + (long) bitWidth * rowCount;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else if (type == 1) {
+ int runCount = ((encoded[scanPos] & 0xFF) << 8) | (encoded[scanPos + 1] & 0xFF);
+ scanPos += 2;
+ long bitPos = ((long) scanPos) * 8L + (long) runCount * bw;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) runCount * bitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else {
+ int cardinality = ((encoded[scanPos] & 0xFF) << 8) | (encoded[scanPos + 1] & 0xFF);
+ scanPos += 2;
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ long bitPos = ((long) scanPos) * 8L + (long) cardinality * bitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) rowCount * dictBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ }
+ }
+ return scanPos;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateSmallerTest.java
new file mode 100644
index 0000000..a620279
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateSmallerTest.java
@@ -0,0 +1,122 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class SubcolumnAddDictPruneNewUpdateSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_update_smaller.csv";
+
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnAddDictPruneNewUpdateTestUtil.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.DataStats stats =
+ SubcolumnAddDictPruneNewUpdateTestUtil.loadDatasetAsIntArray(file);
+ int[] origin = stats.values;
+ if (origin.length == 0) {
+ continue;
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ long start = System.nanoTime();
+ int encodedLength = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = SubcolumnPruneNewTest.Encoder(origin, blockSize, encodedResult);
+
+ SubcolumnAddDictPruneNewUpdateTestUtil.TailInfo tailInfo =
+ SubcolumnAddDictPruneNewUpdateTestUtil.locateTailInfo(encodedResult, encodedLength);
+ if (tailInfo.remainder <= 0) {
+ continue;
+ }
+
+ int updateIndex = tailInfo.numBlocks * blockSize + tailInfo.remainder - 1;
+ int updatedValue = (stats.minValue == Integer.MIN_VALUE) ? stats.minValue : stats.minValue - 1;
+ int[] updated = new int[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ updated[updateIndex] = updatedValue;
+
+ long updateCompressTime;
+ int updatedLength = encodedLength;
+ int[] beta = new int[] {2};
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos = tailInfo.tailStartPos;
+ if (tailInfo.remainder <= 3) {
+ int base = tailInfo.numBlocks * blockSize;
+ for (int i = 0; i < tailInfo.remainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(updated[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ SubcolumnPruneNewTest.BlockEncoder(
+ updated,
+ tailInfo.numBlocks,
+ blockSize,
+ tailInfo.remainder,
+ encodePos,
+ encodedResult,
+ beta);
+ }
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ updateCompressTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(updateCompressTime),
+ String.valueOf(origin.length),
+ String.valueOf(tailInfo.remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateTestUtil.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateTestUtil.java
new file mode 100644
index 0000000..1c20e42
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneNewUpdateTestUtil.java
@@ -0,0 +1,161 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+class SubcolumnAddDictPruneNewUpdateTestUtil {
+
+ static class TailInfo {
+ int tailStartPos;
+ int numBlocks;
+ int remainder;
+ }
+
+ static class DataStats {
+ int[] values;
+ int minValue;
+ int maxValue;
+ }
+
+ private SubcolumnAddDictPruneNewUpdateTestUtil() {}
+
+ static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.length() - decimalIndex - 1;
+ }
+
+ static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ static DataStats loadDatasetAsIntArray(File file) throws IOException {
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ int maxMul = (int) Math.pow(10, maxDecimal);
+
+ DataStats stats = new DataStats();
+ stats.values = new int[data.size()];
+ stats.minValue = Integer.MAX_VALUE;
+ stats.maxValue = Integer.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ stats.values[i] = (int) (data.get(i) * maxMul);
+ if (stats.values[i] < stats.minValue) {
+ stats.minValue = stats.values[i];
+ }
+ if (stats.values[i] > stats.maxValue) {
+ stats.maxValue = stats.values[i];
+ }
+ }
+ if (data.isEmpty()) {
+ stats.minValue = 0;
+ stats.maxValue = 0;
+ }
+ return stats;
+ }
+
+ private static int skipBlock(byte[] encodedResult, int encodePos, int blockSize, int rowCount) {
+ encodePos += 4;
+ int m = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int beta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int bw = SubcolumnPruneNewTest.bitWidth(blockSize);
+ int scanPos = encodePos;
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ long bitPos = ((long) scanPos) * 8L + (long) bitWidth * rowCount;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else if (type == 1) {
+ int runCount = ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ scanPos += 2;
+ long bitPos = ((long) scanPos) * 8L + (long) runCount * bw;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) runCount * bitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else {
+ int cardinality =
+ ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ scanPos += 2;
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ long bitPos = ((long) scanPos) * 8L + (long) cardinality * bitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) rowCount * dictBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ }
+ }
+ return scanPos;
+ }
+
+ static TailInfo locateTailInfo(byte[] encodedResult, int encodedLength) {
+ TailInfo info = new TailInfo();
+ int encodePos = 0;
+ int dataLength = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int blockSize = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ info.numBlocks = numBlocks;
+ info.remainder = remainder;
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = skipBlock(encodedResult, encodePos, blockSize, blockSize);
+ }
+ info.tailStartPos = (remainder > 0) ? encodePos : encodedLength;
+ return info;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneTimeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneTimeTest.java
new file mode 100644
index 0000000..2b7c120
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictPruneTimeTest.java
@@ -0,0 +1,1025 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Stream;
+
+public class SubcolumnAddDictPruneTimeTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size, int[] encodingType) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int[] threshold = null;
+
+ switch(block_size) {
+ case 32:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ case 64:
+ threshold = new int[] {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ break;
+ case 128:
+ threshold = new int[] {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ break;
+ case 256:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ break;
+ case 512:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ break;
+ case 1024:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ break;
+ case 2048:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ break;
+ case 4096:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ break;
+ case 8192:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ break;
+ default:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ }
+
+ int cost1 = 0;
+
+ // System.out.println("x:");
+ // for (int i = 0; i < x_length; i++) {
+ // System.out.print(x[i] + " ");
+ // }
+ // System.out.println();
+
+ BitSet[] bitsets = new BitSet[m];
+
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+
+ for (int i = 0; i < m; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int current_value = (x[0] >> i) & 1;
+
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ int count = 0;
+
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+
+ for (int j = 1; j < x_length; j++) {
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+
+ bitsets[i].set(j - 1);
+ }
+
+ }
+
+ bitsets[i].set(x_length - 1);
+
+ count++;
+
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+
+ if (bpe_cost_single[i] <= rle_cost_single[i] && bpe_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 0; // bpe
+ cost1 += bpe_cost_single[i];
+ } else if (rle_cost_single[i] < bpe_cost_single[i] && rle_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 1; // rle
+ cost1 += rle_cost_single[i];
+ } else {
+ encodingType[i] = 2; // de
+ cost1 += de_cost_single[i];
+ }
+
+ }
+
+ int cMin = cost1;
+
+ int[] beta_list = new int[m - 1];
+ for (int i = 0; i < m - 1; i++) {
+ beta_list[i] = i + 2;
+ }
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int cost = 0;
+
+ int[] encodingTypeTemp = new int[l];
+
+ for (int i = 0; i < l; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int currentCost = 0;
+
+ int bpCost = 0;
+
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start >= i * beta && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+
+ if (beta_start < i * beta) {
+ beta_start = i * beta;
+ }
+
+ bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
+ // System.out.println("bpCost: " + bpCost);
+
+ currentCost = bpCost;
+
+ int rleCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ // if (rle_cost_single[i * beta] < currentCost) {
+ int rleCost = 0;
+
+ boolean currentBetter = false;
+
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (currentCost > rleCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ }
+
+ int deCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (de_cost_single[j] > deCostMax) {
+ deCostMax = de_cost_single[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ // if (de_cost_single[i * beta] < currentCost) {
+ boolean currentBetter = false;
+ Set<Integer> uniqueValues = new HashSet<>();
+
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+
+ if (deCost < currentCost) {
+ currentCost = deCost;
+
+ encodingTypeTemp[i] = 2;
+ }
+ }
+
+ }
+
+ cost += currentCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ // System.out.println("betaBest: " + betaBest);
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size, int[] encodingType) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ // System.out.println("maxValue: " + maxValue);
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ // System.out.println("All zero list.");
+ return encode_pos;
+ }
+
+ int l;
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+
+ if (encodingType[i] == 2) {
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ int dict_bit_width = bitWidth(cardinality) ;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ }
+
+ index++;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if(type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ encode_pos =decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta, long[] betaTime,
+ long[] encodeTime) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ int[] encodingType = new int[m];
+
+ long s1 = System.nanoTime();
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size, encodingType);
+ long e1 = System.nanoTime();
+ betaTime[0] += (e1 - s1);
+
+
+ long s2 = System.nanoTime();
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size, encodingType);
+ long e2 = System.nanoTime();
+ encodeTime[0] += (e2 - s2);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, long[] betaTime,
+ long[] encodeTime) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta, betaTime, encodeTime);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta, betaTime, encodeTime);
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D://github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_dictionary2.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Beta Selection Time",
+ "Subcolumn Encode Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long betaTime = 0;
+ long subcolumnEncodeTime = 0;
+
+ int length = 0;
+
+ long[] betaTimeArr = new long[1];
+ long[] subcolumnEncodeTimeArr = new long[1];
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+
+ betaTimeArr[0] = 0;
+ subcolumnEncodeTimeArr[0] = 0;
+ length = Encoder(data2_arr, block_size, encoded_result, betaTimeArr, subcolumnEncodeTimeArr);
+
+ betaTime += betaTimeArr[0];
+ subcolumnEncodeTime += subcolumnEncodeTimeArr[0];
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ betaTime /= repeatTime;
+ subcolumnEncodeTime /= repeatTime;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(betaTime),
+ String.valueOf(subcolumnEncodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGreaterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGreaterTest.java
new file mode 100644
index 0000000..4f2c21e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGreaterTest.java
@@ -0,0 +1,372 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+public class SubcolumnAddDictQueryGreaterTest {
+
+ public static void Query(byte[] encodedResult, int lowerBound) {
+ int encodePos = 0;
+
+ int dataLength = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] result = new int[dataLength];
+ int[] resultLength = new int[1];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockQueryIndex(
+ encodedResult, i, blockSize, blockSize, encodePos, lowerBound, result, resultLength);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int value = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ if (value > lowerBound) {
+ result[resultLength[0]++] = base + i;
+ }
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockQueryIndex(
+ encodedResult,
+ numBlocks,
+ blockSize,
+ remainder,
+ encodePos,
+ lowerBound,
+ result,
+ resultLength);
+ }
+ }
+
+ public static int BlockQueryIndex(
+ byte[] encodedResult,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ int lowerBound,
+ int[] result,
+ int[] resultLength) {
+ int minDelta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int adjustedLower = lowerBound - minDelta;
+ int base = blockIndex * blockSize;
+
+ int m = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ if (m == 0) {
+ if (adjustedLower < 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[resultLength[0]++] = base + i;
+ }
+ }
+ return encodePos;
+ }
+
+ int beta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+ int[] encodingType = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int bw = SubcolumnPruneNewTest.bitWidth(blockSize);
+
+ int[] segmentPos = new int[l];
+ int[] runCountList = new int[l];
+ int[] cardinalityList = new int[l];
+ int scanPos = encodePos;
+
+ // Payload is encoded in i=0..l-1 order. First pass records offsets and computes end position.
+ for (int i = 0; i < l; i++) {
+ segmentPos[i] = scanPos;
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+
+ if (type == 0) {
+ long bitPos = ((long) scanPos) * 8L + (long) currentBitWidth * remainder;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else if (type == 1) {
+ int runCount =
+ ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ runCountList[i] = runCount;
+ scanPos += 2;
+
+ long bitPos = ((long) scanPos) * 8L + (long) runCount * bw;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) runCount * currentBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else {
+ int cardinality =
+ ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ cardinalityList[i] = cardinality;
+ scanPos += 2;
+
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ long bitPos = ((long) scanPos) * 8L + (long) cardinality * currentBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) remainder * dictBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ }
+ }
+
+ if (adjustedLower < 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[resultLength[0]++] = base + i;
+ }
+ return scanPos;
+ }
+
+ int[] candidateIndices = new int[remainder];
+ int candidateLength = remainder;
+ for (int i = 0; i < remainder; i++) {
+ candidateIndices[i] = i;
+ }
+
+ // Query compares from high to low subcolumn.
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+ int boundPart = (adjustedLower >> (i * beta)) & ((1 << beta) - 1);
+
+ if (type == 0) {
+ int pos = segmentPos[i];
+ long bitStart = ((long) pos) * 8L;
+
+ int newLength = 0;
+ for (int j = 0; j < candidateLength; j++) {
+ int index = candidateIndices[j];
+ int current =
+ SubcolumnPruneNewTest.bytesToInt(
+ encodedResult, (int) (bitStart + (long) index * currentBitWidth), currentBitWidth);
+ if (current > boundPart) {
+ result[resultLength[0]++] = base + index;
+ } else if (current == boundPart) {
+ candidateIndices[newLength++] = index;
+ }
+ }
+ candidateLength = newLength;
+ } else if (type == 1) {
+ int pos = segmentPos[i];
+ int runCount = runCountList[i];
+ pos += 2;
+
+ int[] runEnd = new int[runCount];
+ int[] rleValues = new int[runCount];
+ pos = SubcolumnPruneNewTest.decodeBitPacking(encodedResult, pos, bw, runCount, runEnd);
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, pos, currentBitWidth, runCount, rleValues);
+
+ int newLength = 0;
+ int runIdx = 0;
+ for (int j = 0; j < candidateLength; j++) {
+ int index = candidateIndices[j];
+ while (runIdx < runCount && runEnd[runIdx] <= index) {
+ runIdx++;
+ }
+ if (runIdx >= runCount) {
+ break;
+ }
+ int current = rleValues[runIdx];
+ if (current > boundPart) {
+ result[resultLength[0]++] = base + index;
+ } else if (current == boundPart) {
+ candidateIndices[newLength++] = index;
+ }
+ }
+ candidateLength = newLength;
+ } else {
+ int pos = segmentPos[i];
+ int cardinality = cardinalityList[i];
+ pos += 2;
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+
+ int[] dictKeyList = new int[cardinality];
+ int[] dictIndexes = new int[remainder];
+ pos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, pos, currentBitWidth, cardinality, dictKeyList);
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, pos, dictBitWidth, remainder, dictIndexes);
+
+ int newLength = 0;
+ for (int j = 0; j < candidateLength; j++) {
+ int index = candidateIndices[j];
+ int current = dictKeyList[dictIndexes[index]];
+ if (current > boundPart) {
+ result[resultLength[0]++] = base + index;
+ } else if (current == boundPart) {
+ candidateIndices[newLength++] = index;
+ }
+ }
+ candidateLength = newLength;
+ }
+ }
+
+ return scanPos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_query_greater.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+ System.out.println("Output: " + outputPath);
+ System.out.println("Block size: " + blockSize);
+ System.out.println("Repeat time: " + repeatTime);
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ System.out.println("maxDecimal: " + maxDecimal);
+
+ int[] dataArr = new int[data.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[dataArr.length * 8];
+ int length = 0;
+
+ long encodeTime;
+ long queryTime;
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnPruneNewTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ encodeTime = (end - start) / repeatTime;
+
+ System.out.println("Query");
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Query(encodedResult, queryRange.get(datasetName));
+ }
+ end = System.nanoTime();
+ queryTime = (end - start) / repeatTime;
+
+ double compressionRatio = length / (double) (data.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ System.out.println("compressionRatio: " + compressionRatio);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGroupTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGroupTest.java
new file mode 100644
index 0000000..39282a9
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnAddDictQueryGroupTest.java
@@ -0,0 +1,523 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+public class SubcolumnAddDictQueryGroupTest {
+
+ private static final int[] BLOCK_SIZES = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+ private static final int TARGET_GROUP_COUNT = 20;
+
+ private static class BlockMeta {
+ int minDelta;
+ int m;
+ int beta;
+ int l;
+ int[] bitWidthList;
+ int[] encodingType;
+ int[] segmentPos;
+ int[] runCountList;
+ int[] cardinalityList;
+ int[][] rleRunEndList;
+ int[][] rleValueList;
+ int[][] dictKeyList;
+ int[] dictBitWidthList;
+ int[] dictIndexPosList;
+ int nextPos;
+ }
+
+ private static class RangeGroupConfig {
+ int start;
+ int width;
+ int groupCount;
+ }
+
+ public static int[] queryGroupCountByValueRange(
+ byte[] encodedResult, int rangeStart, int rangeWidth, int bucketCount) {
+ int encodePos = 0;
+ int dataLength = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int blockSize = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int encodedBlocks = numBlocks + (remainder > 3 ? 1 : 0);
+
+ BlockMeta[] metas = new BlockMeta[encodedBlocks];
+ int[] blockRowCount = new int[encodedBlocks];
+ for (int i = 0; i < encodedBlocks; i++) {
+ int rowCount = (i < numBlocks) ? blockSize : remainder;
+ if (rowCount == 0) {
+ break;
+ }
+ blockRowCount[i] = rowCount;
+ BlockMeta meta = parseBlockMeta(encodedResult, encodePos, blockSize, rowCount);
+ metas[i] = meta;
+ encodePos = meta.nextPos;
+ }
+
+ int[] groupCounts = new int[bucketCount];
+ long rangeEnd = rangeStart + (long) rangeWidth * bucketCount;
+ for (int b = 0; b < encodedBlocks; b++) {
+ accumulateBlockRangeCounts(
+ encodedResult,
+ metas[b],
+ blockRowCount[b],
+ rangeStart,
+ rangeWidth,
+ bucketCount,
+ rangeEnd,
+ groupCounts);
+ }
+
+ if (remainder > 0 && remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int bucket = bucketIndex(value, rangeStart, rangeWidth, bucketCount);
+ if (bucket >= 0) {
+ groupCounts[bucket]++;
+ }
+ }
+ }
+
+ return groupCounts;
+ }
+
+ private static void accumulateBlockRangeCounts(
+ byte[] encodedResult,
+ BlockMeta meta,
+ int rowCount,
+ int rangeStart,
+ int rangeWidth,
+ int bucketCount,
+ long rangeEnd,
+ int[] groupCounts) {
+ if (rowCount <= 0) {
+ return;
+ }
+ if (meta.m == 0) {
+ int bucket = bucketIndex(meta.minDelta, rangeStart, rangeWidth, bucketCount);
+ if (bucket >= 0) {
+ groupCounts[bucket] += rowCount;
+ }
+ return;
+ }
+
+ int[] candidates = new int[rowCount];
+ for (int i = 0; i < rowCount; i++) {
+ candidates[i] = i;
+ }
+
+ accumulateRangeByPrefix(
+ encodedResult,
+ meta,
+ rangeStart,
+ rangeWidth,
+ bucketCount,
+ rangeEnd,
+ groupCounts,
+ candidates,
+ rowCount,
+ meta.l - 1,
+ 0L);
+ }
+
+ private static void accumulateRangeByPrefix(
+ byte[] encodedResult,
+ BlockMeta meta,
+ int rangeStart,
+ int rangeWidth,
+ int bucketCount,
+ long rangeEnd,
+ int[] groupCounts,
+ int[] candidates,
+ int candidateLength,
+ int level,
+ long prefix) {
+ if (candidateLength <= 0) {
+ return;
+ }
+
+ int remainingBits = Math.max(0, level * meta.beta);
+ long prefixBase = prefix << remainingBits;
+ long suffixMax = (remainingBits == 0) ? 0L : ((1L << remainingBits) - 1L);
+ long lowValue = meta.minDelta + prefixBase;
+ long highValue = meta.minDelta + prefixBase + suffixMax;
+
+ if (highValue < rangeStart || lowValue >= rangeEnd) {
+ return;
+ }
+
+ int startBucket = (int) Math.floorDiv(lowValue - (long) rangeStart, rangeWidth);
+ int endBucket = (int) Math.floorDiv(highValue - (long) rangeStart, rangeWidth);
+ if (startBucket == endBucket && startBucket >= 0 && startBucket < bucketCount) {
+ groupCounts[startBucket] += candidateLength;
+ return;
+ }
+
+ if (level < 0) {
+ int bucket = bucketIndex(lowValue, rangeStart, rangeWidth, bucketCount);
+ if (bucket >= 0) {
+ groupCounts[bucket] += candidateLength;
+ }
+ return;
+ }
+
+ int radix = 1 << meta.beta;
+ int[] partCounts = new int[radix];
+ int[] partsPerCandidate = new int[candidateLength];
+ for (int i = 0; i < candidateLength; i++) {
+ int part = partValueAtIndex(encodedResult, meta, level, candidates[i]);
+ partsPerCandidate[i] = part;
+ partCounts[part]++;
+ }
+
+ int[][] partCandidates = new int[radix][];
+ for (int part = 0; part < radix; part++) {
+ if (partCounts[part] > 0) {
+ partCandidates[part] = new int[partCounts[part]];
+ }
+ }
+
+ int[] offsets = new int[radix];
+ for (int i = 0; i < candidateLength; i++) {
+ int part = partsPerCandidate[i];
+ partCandidates[part][offsets[part]++] = candidates[i];
+ }
+
+ for (int part = 0; part < radix; part++) {
+ int length = partCounts[part];
+ if (length == 0) {
+ continue;
+ }
+ long childPrefix = (prefix << meta.beta) | part;
+ accumulateRangeByPrefix(
+ encodedResult,
+ meta,
+ rangeStart,
+ rangeWidth,
+ bucketCount,
+ rangeEnd,
+ groupCounts,
+ partCandidates[part],
+ length,
+ level - 1,
+ childPrefix);
+ }
+ }
+
+ private static int partValueAtIndex(
+ byte[] encodedResult, BlockMeta meta, int level, int localIndex) {
+ int type = meta.encodingType[level];
+ int currentBitWidth = meta.bitWidthList[level];
+ if (type == 0) {
+ long bitStart = ((long) meta.segmentPos[level]) * 8L;
+ return SubcolumnPruneNewTest.bytesToInt(
+ encodedResult, (int) (bitStart + (long) localIndex * currentBitWidth), currentBitWidth);
+ }
+ if (type == 1) {
+ int[] runEnd = meta.rleRunEndList[level];
+ int[] rleValues = meta.rleValueList[level];
+ int runIndex = Arrays.binarySearch(runEnd, localIndex + 1);
+ if (runIndex < 0) {
+ runIndex = -runIndex - 1;
+ }
+ if (runIndex < 0 || runIndex >= rleValues.length) {
+ return 0;
+ }
+ return rleValues[runIndex];
+ }
+ int dictIndex =
+ bitPackedValueAt(
+ encodedResult, meta.dictIndexPosList[level], meta.dictBitWidthList[level], localIndex);
+ return meta.dictKeyList[level][dictIndex];
+ }
+
+ private static int bucketIndex(long value, int rangeStart, int rangeWidth, int bucketCount) {
+ long idx = Math.floorDiv(value - (long) rangeStart, rangeWidth);
+ if (idx < 0 || idx >= bucketCount) {
+ return -1;
+ }
+ return (int) idx;
+ }
+
+ private static RangeGroupConfig buildRangeGroupConfig(int[] dataArr) {
+ int min = Integer.MAX_VALUE;
+ int max = Integer.MIN_VALUE;
+ for (int value : dataArr) {
+ if (value < min) {
+ min = value;
+ }
+ if (value > max) {
+ max = value;
+ }
+ }
+
+ long range = (long) max - min + 1L;
+ int width = (int) Math.max(1L, (range + TARGET_GROUP_COUNT - 1L) / TARGET_GROUP_COUNT);
+ // int nice = niceWidth(width);
+ // Keep bucket count as close as possible to TARGET_GROUP_COUNT.
+ int groupCount = TARGET_GROUP_COUNT;
+
+ RangeGroupConfig config = new RangeGroupConfig();
+ config.start = min;
+ config.width = width;
+ config.groupCount = groupCount;
+ return config;
+ }
+
+ /*
+ private static int niceWidth(int rawWidth) {
+ int scale = 1;
+ while (rawWidth >= 10) {
+ rawWidth = (rawWidth + 9) / 10;
+ scale *= 10;
+ }
+ if (rawWidth <= 1) {
+ return scale;
+ }
+ if (rawWidth <= 2) {
+ return 2 * scale;
+ }
+ if (rawWidth <= 5) {
+ return 5 * scale;
+ }
+ return 10 * scale;
+ }
+ */
+
+ private static BlockMeta parseBlockMeta(byte[] encodedResult, int encodePos, int blockSize, int rowCount) {
+ BlockMeta meta = new BlockMeta();
+ meta.minDelta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ meta.m = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (meta.m == 0) {
+ meta.nextPos = encodePos;
+ return meta;
+ }
+
+ meta.beta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ meta.l = (meta.m + meta.beta - 1) / meta.beta;
+
+ meta.bitWidthList = new int[meta.l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, encodePos, 8, meta.l, meta.bitWidthList);
+ meta.encodingType = new int[meta.l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, encodePos, 2, meta.l, meta.encodingType);
+
+ int bw = SubcolumnPruneNewTest.bitWidth(blockSize);
+ meta.segmentPos = new int[meta.l];
+ meta.runCountList = new int[meta.l];
+ meta.cardinalityList = new int[meta.l];
+ meta.rleRunEndList = new int[meta.l][];
+ meta.rleValueList = new int[meta.l][];
+ meta.dictKeyList = new int[meta.l][];
+ meta.dictBitWidthList = new int[meta.l];
+ meta.dictIndexPosList = new int[meta.l];
+ int scanPos = encodePos;
+
+ for (int i = 0; i < meta.l; i++) {
+ meta.segmentPos[i] = scanPos;
+ int type = meta.encodingType[i];
+ int currentBitWidth = meta.bitWidthList[i];
+ if (type == 0) {
+ long bitPos = ((long) scanPos) * 8L + (long) currentBitWidth * rowCount;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else if (type == 1) {
+ int runCount = ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ meta.runCountList[i] = runCount;
+ scanPos += 2;
+ int[] runEnd = new int[runCount];
+ scanPos = SubcolumnPruneNewTest.decodeBitPacking(encodedResult, scanPos, bw, runCount, runEnd);
+ int[] rleValues = new int[runCount];
+ scanPos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, scanPos, currentBitWidth, runCount, rleValues);
+ meta.rleRunEndList[i] = runEnd;
+ meta.rleValueList[i] = rleValues;
+ } else {
+ int cardinality = ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ meta.cardinalityList[i] = cardinality;
+ scanPos += 2;
+ int[] dictKey = new int[cardinality];
+ int dictIndexPos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, scanPos, currentBitWidth, cardinality, dictKey);
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ long bitPos = ((long) dictIndexPos) * 8L + (long) rowCount * dictBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ meta.dictKeyList[i] = dictKey;
+ meta.dictBitWidthList[i] = dictBitWidth;
+ meta.dictIndexPosList[i] = dictIndexPos;
+ }
+ }
+
+ meta.nextPos = scanPos;
+ return meta;
+ }
+
+ private static int bitPackedValueAt(
+ byte[] encodedResult, int bitPackedStartBytePos, int bitWidth, int index) {
+ long bitPos = ((long) bitPackedStartBytePos) * 8L + (long) index * bitWidth;
+ return SubcolumnPruneNewTest.bytesToInt(encodedResult, (int) bitPos, bitWidth);
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_query_group_range_count.csv";
+
+ int repeatTime = 100;
+ System.out.println("Output: " + outputPath);
+ System.out.println("Block sizes: " + java.util.Arrays.toString(BLOCK_SIZES));
+ System.out.println("Repeat time: " + repeatTime);
+ System.out.println("Target group count: " + TARGET_GROUP_COUNT);
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Block Size",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ System.out.println("maxDecimal: " + maxDecimal);
+
+ int[] dataArr = new int[data.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+ RangeGroupConfig groupConfig = buildRangeGroupConfig(dataArr);
+ System.out.println(
+ "group range config: start="
+ + groupConfig.start
+ + ", width="
+ + groupConfig.width
+ + ", groupCount="
+ + groupConfig.groupCount);
+
+ for (int blockSize : BLOCK_SIZES) {
+ byte[] encodedResult = new byte[dataArr.length * 8];
+ int length = 0;
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnPruneNewTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ System.out.println("RangeGroupCountQuery");
+ int[] groupCounts = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ groupCounts =
+ queryGroupCountByValueRange(
+ encodedResult, groupConfig.start, groupConfig.width, groupConfig.groupCount);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+
+ System.out.println(
+ "blockSize="
+ + blockSize
+ + ", groupCount: "
+ + (groupCounts == null ? 0 : groupCounts.length));
+ double compressionRatio = length / (double) (data.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "SubcolumnAddDictPruneNew",
+ String.valueOf(blockSize),
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ System.out.println("compressionRatio: " + compressionRatio);
+ }
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPBetaTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPBetaTest.java
new file mode 100644
index 0000000..3724d86
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPBetaTest.java
@@ -0,0 +1,175 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnBPBetaTest {
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ encoded_result[encode_pos] = (byte) (min_delta[0] >> 24);
+ encoded_result[encode_pos + 1] = (byte) (min_delta[0] >> 16);
+ encoded_result[encode_pos + 2] = (byte) (min_delta[0] >> 8);
+ encoded_result[encode_pos + 3] = (byte) min_delta[0];
+ encode_pos += 4;
+
+ encode_pos = SubcolumnBPTest.SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnBPTest.SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int beta_value) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = beta_value;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ encoded_result[encode_pos] = (byte) (value >> 24);
+ encoded_result[encode_pos + 1] = (byte) (value >> 16);
+ encoded_result[encode_pos + 2] = (byte) (value >> 8);
+ encoded_result[encode_pos + 3] = (byte) value;
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPTest.java
new file mode 100644
index 0000000..230fff8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBPTest.java
@@ -0,0 +1,840 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnBPTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnBP(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ // int rleCost = 0;
+
+ // // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+
+ // int index = 0;
+
+ // boolean bpBest = false;
+
+ // for (int j = 1; j < x_length; j++) {
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ // }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // index++;
+
+ // System.out.println("index: " + index);
+
+ // rleCost = bw * index + bitWidthListList[i] * index;
+
+ // // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += bpCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // int bpCost = bitWidthList[i] * list_length;
+ // int rleCost = 0;
+
+ // int previous = subcolumnList[i][0];
+ // int index = 0;
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // index++;
+ // previous = currentNumber;
+ // }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ // }
+
+ // index++;
+
+ // rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ // encodingType[i] = 1;
+
+ // encoded_result[encode_pos] = (byte) (index >> 8);
+ // encode_pos += 1;
+ // encoded_result[encode_pos] = (byte) (index & 0xFF);
+ // encode_pos += 1;
+
+ // index = 0;
+ // int[] run_length = new int[list_length];
+ // int[] rle_values = new int[list_length];
+ // previous = subcolumnList[i][0];
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // run_length[index] = j;
+ // rle_values[index] = previous;
+ // index++;
+ // previous = currentNumber;
+ // }
+ // }
+
+ // run_length[index] = list_length;
+ // rle_values[index] = previous;
+ // index++;
+
+ // encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ // encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ // if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ // } else {
+ // int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ // encode_pos += 2;
+
+ // int[] run_length = new int[index];
+ // int[] rle_values = new int[index];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ // int currentIndex = 0;
+ // for (int j = 0; j < index; j++) {
+ // int endPos = run_length[j];
+ // int value = rle_values[j];
+ // while (currentIndex < endPos) {
+ // subcolumnList[i][currentIndex] = value;
+ // currentIndex++;
+ // }
+ // }
+ // }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = SubcolumnBP(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ // String input_parent_dir = parent_dir + "dataset/CMS9";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "subcolumn.csv";
+ String outputPath = output_parent_dir + "subcolumn_bp.csv";
+
+ // int block_size = 512;
+ int block_size = 256;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBetaTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBetaTest.java
index 5a4ed89..b9ea144 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBetaTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBetaTest.java
@@ -16,8 +16,6 @@
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
-import static org.junit.Assert.assertEquals;
-
public class SubcolumnBetaTest {
public static int[] getAbsDeltaTsBlock(
@@ -173,15 +171,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -203,26 +198,25 @@
}
@Test
- public void testSubcolumn() throws IOException {
+ public void test0() throws IOException {
+ // String parent_dir = "path/to/your/directory/";
String parent_dir = "D:/github/xjz17/subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/compression_vs_beta/";
- // String output_parent_dir = parent_dir + "result/compression_vs_beta/";
+ String output_parent_dir = parent_dir + "result/compression_vs_beta/";
+ File outputDir = new File(output_parent_dir);
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ }
+
int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31 };
- // int block_size = 1024;
int block_size = 512;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 500;
for (int beta : beta_list) {
@@ -243,7 +237,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -275,7 +268,7 @@
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
@@ -296,11 +289,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -317,10 +306,6 @@
e = System.nanoTime();
decodeTime += ((e - s) / repeatTime);
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
-
String[] record = {
datasetName,
"Sub-columns",
@@ -339,164 +324,4 @@
}
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/compression_vs_beta/";
- // String output_parent_dir = parent_dir + "trans_data_result/compression_vs_beta/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- // for (String name : dataset_name) {
- // output_path_list.add(output_parent_dir + name + "_ratio.csv");
- // }
-
- for (int beta : beta_list) {
-
- String outputPath = output_parent_dir + "subcolumn_trans_data_beta_" + beta + ".csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- // CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
- // writer.setRecordDelimiter('\n');
-
- // String[] head = {
- // "Input Direction",
- // "Encoding Algorithm",
- // "Encoding Time",
- // "Decoding Time",
- // "Points",
- // "Compressed Size",
- // "Compression Ratio"
- // };
- // writer.writeRecord(head);
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
-
- System.out.println("beta: " + beta);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBlockSizeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBlockSizeTest.java
index 328749f..fdff3ed 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBlockSizeTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnBlockSizeTest.java
@@ -51,8 +51,9 @@
}
@Test
- public void testSubcolumn() throws IOException {
+ public void test0() throws IOException {
String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
@@ -61,13 +62,11 @@
int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
- int repeatTime = 100;
+ // int repeatTime = 100;
+ int repeatTime = 500;
// repeatTime = 1;
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
for (int block_size : block_size_list) {
String outputPath = output_parent_dir + "subcolumn_block_" + block_size + ".csv";
@@ -139,11 +138,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -161,7 +156,7 @@
decodeTime += ((e - s) / repeatTime);
for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
}
String[] record = {
@@ -182,140 +177,4 @@
}
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/compression_vs_block/";
- // String output_parent_dir = parent_dir + "trans_data_result/compression_vs_block/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024); // Default block size, can be changed if needed
- });
- }
-
- for (int block_size : block_size_list) {
-
- String outputPath = output_parent_dir + "subcolumn_trans_data_block_" + block_size + ".csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
- inputStream.close();
- int[] data2_arr = new int[data2.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = SubcolumnTest.Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
-
- System.out.println("Block size: " + block_size);
- }
-
- writer.close();
- }
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnCountWithNULLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnCountWithNULLTest.java
new file mode 100644
index 0000000..fefa31b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnCountWithNULLTest.java
@@ -0,0 +1,501 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class SubcolumnCountWithNULLTest {
+
+ public static void Query(byte[] encoded_result, int target) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryCount(encoded_result, i, block_size,
+ block_size, encode_pos, target,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ if (value == target) {
+ result[result_length[0]]++;
+ }
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockQueryCount(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, target,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryCount(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int target, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ target -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 target 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ result[result_length[0]] += remainder;
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ int current = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][index] < value) {
+ // result[result_length[0]] = block_size * block_index + index;
+ // result_length[0]++;
+ // } else if (subcolumnList[i][index] == value) {
+ // candidate_indices[new_length] = index;
+ // new_length++;
+ // }
+ if (current == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ // if (rle_values[rleIndex] < value) {
+ // result[result_length[0]] = block_size * block_index + index_candidate;
+ // result_length[0]++;
+ // } else if (rle_values[rleIndex] == value) {
+ // candidate_indices[new_length] = index_candidate;
+ // new_length++;
+ // }
+ if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ // if (target <= 0) {
+ // for (int i = 0; i < remainder; i++) {
+ // result[result_length[0]] = block_size * block_index + i;
+ // result_length[0]++;
+ // }
+ // return encode_pos;
+ // }
+
+ result[result_length[0]] += candidate_length;
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ @Test
+ public void testQueryBeta() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/query_count_null/"; //""D:/encoding-subcolumn/result/";
+
+
+
+// int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+// 24, 25, 26, 27, 28, 29, 30, 31 };
+ double[] null_rate_list = {0,0.5};//,0.1,0.2,0.3,0.4,0.6,0.7,0.8,0.9,1
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (int x =1;x>=0 ; x--) {
+ double null_rate=null_rate_list[x];
+ String outputPath = output_parent_dir + "subcolumn_query_count_null_" + null_rate + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int nullCountPerBlock = (int) (null_rate * (double) block_size); // 每块中要设置为null的数量
+ int new_arr_length = (int) ((double)(data1.size()/block_size*block_size)*(1-null_rate)+
+ (double)(data1.size()-data1.size()/block_size*block_size)*(1-null_rate));
+ System.out.println("new_arr_length:"+new_arr_length);
+ int[] data2_arr_new = new int[new_arr_length];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int all_data_size = data1.size();
+
+ java.util.BitSet bitmap = new java.util.BitSet(data1.size());
+ int new_array_index = 0;
+
+ if(null_rate==1){
+ int[] bitmap_bit = new int[data1.size()];
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ for(int i=0;i<data1.size();i++){
+ bitmap_bit[i] = 0;
+ }
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+ double ratioTmp = 0;
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ String[] decode_values = new String[data1.size()];
+ for(int i=0;i<data1.size();i++){
+ decode_values[i] = "NULL";
+ }
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ continue;
+ } else if(null_rate != 0 && null_rate != 1){
+ for (int blockStart = 0; blockStart < data1.size(); blockStart += block_size) {
+ int blockEnd = Math.min(blockStart + block_size, data1.size());
+ int actualBlockSize = (int) ((double)(blockEnd - blockStart)*(1-null_rate));
+ int actualNullCount = Math.min(nullCountPerBlock, actualBlockSize);
+
+ // 创建当前块的索引列表用于随机选择
+ java.util.List<Integer> indices = new java.util.ArrayList<>();
+ for (int i = blockStart; i < blockEnd; i++) {
+ indices.add(i);
+ }
+
+ // 随机打乱并选择要设置为null的位置
+ java.util.Collections.shuffle(indices);
+ java.util.List<Integer> selectedIndices = new java.util.ArrayList<>();
+ for (int i = 0; i < actualNullCount; i++) {
+ selectedIndices.add(indices.get(i));
+ }
+ java.util.Collections.sort(selectedIndices);
+ int i = 0;
+ int j = blockStart;
+// int nullIndex;
+ while (j < blockEnd) {
+ if (i < actualNullCount && j == selectedIndices.get(i)) {
+ // 这个位置被移除，设置bitmap并跳过
+ bitmap.set(j);
+ i++;
+ } else {
+ // 这个位置保留，复制到新数组
+// if(new_array_index==70000){
+// System.out.println(j/block_size*block_size);
+// System.out.println(j);
+// System.out.println(new_array_index);
+// }
+ data2_arr_new[new_array_index] = data2_arr[j];
+ new_array_index++;
+ if(new_array_index == new_arr_length) break;
+ }
+ j++;
+ }
+ if(new_array_index == new_arr_length) break;
+ }
+ }else {
+ data2_arr_new = data2_arr;
+ }
+
+// System.out.println(bitmap);
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr_new, (block_size-nullCountPerBlock), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+// SubcolumnCountWithNULLTest.Query(encoded_result, queryRange.get(datasetName));
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int count = bitmap.cardinality();
+// System.out.println(data1.size()-count);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(all_data_size),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullEncodingTypeRatioTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullEncodingTypeRatioTest.java
new file mode 100644
index 0000000..a054975
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullEncodingTypeRatioTest.java
@@ -0,0 +1,572 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class SubcolumnFullEncodingTypeRatioTest {
+
+ private static class EncodingTypeStats {
+ private int totalBlockCount;
+ private int totalSubcolumnCount;
+ private final int[] encodingTypeCounts = new int[3];
+
+ private void setTotalBlockCount(int totalBlockCount) {
+ this.totalBlockCount = totalBlockCount;
+ }
+
+ private void recordEncodingType(int[] encodingType, int length) {
+ if (length <= 0) {
+ return;
+ }
+ totalSubcolumnCount += length;
+ for (int i = 0; i < length; i++) {
+ int currentType = encodingType[i];
+ if (currentType >= 0 && currentType < encodingTypeCounts.length) {
+ encodingTypeCounts[currentType]++;
+ }
+ }
+ }
+
+ private int getTotalBlockCount() {
+ return totalBlockCount;
+ }
+
+ private int getTotalSubcolumnCount() {
+ return totalSubcolumnCount;
+ }
+
+ private int getBitPackingSubcolumnCount() {
+ return encodingTypeCounts[0];
+ }
+
+ private int getRleSubcolumnCount() {
+ return encodingTypeCounts[1];
+ }
+
+ private int getDictionarySubcolumnCount() {
+ return encodingTypeCounts[2];
+ }
+
+ private double getBitPackingRatio() {
+ return getEncodingTypeRatio(0);
+ }
+
+ private double getRleRatio() {
+ return getEncodingTypeRatio(1);
+ }
+
+ private double getDictionaryRatio() {
+ return getEncodingTypeRatio(2);
+ }
+
+ private double getEncodingTypeRatio(int type) {
+ if (totalSubcolumnCount == 0) {
+ return 0;
+ }
+ return encodingTypeCounts[type] / (double) totalSubcolumnCount;
+ }
+ }
+
+ public static int SubcolumnEncoder(
+ int[] list,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ int blockSize,
+ EncodingTypeStats stats) {
+ int listLength = list.length;
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = SubcolumnFullTest.bitWidth(maxValue);
+
+ SubcolumnFullTest.intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int l = (m + beta[0] - 1) / beta[0];
+ int[] bitWidthList = new int[l];
+ int[][] subcolumnList = new int[l][listLength];
+
+ SubcolumnFullTest.intByte2Bytes(beta[0], encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = SubcolumnFullTest.bitWidth(blockSize);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < listLength; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = SubcolumnFullTest.bitWidth(maxValuePart);
+ }
+
+ encodePos = SubcolumnFullTest.bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int[] encodingType = new int[l];
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ for (int i = l - 1; i >= 0; i--) {
+ int bpCost = bitWidthList[i] * listLength;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+ for (int j = 1; j < listLength; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < listLength; j++) {
+ uniqueValues.add(subcolumnList[i][j]);
+ }
+ int cardinality = uniqueValues.size();
+
+ index++;
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (cardinality < Math.pow(2, bitWidthList[i] - 1)) {
+ int dictBitWidth = SubcolumnFullTest.bitWidth(cardinality);
+ int dictCost =
+ dictBitWidth * listLength + cardinality * (bitWidthList[i] + dictBitWidth);
+ if (dictCost < rleCost && dictCost < bpCost) {
+ encodingType[i] = 2;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dictKeyList = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dictKeyList[j] = sortedUnique.get(j);
+ }
+ for (int j = 0; j < listLength; j++) {
+ subcolumnList[i][j] = valueToCode.get(subcolumnList[i][j]);
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos =
+ SubcolumnFullTest.bitPacking(
+ dictKeyList, bitWidthList[i], encodePos, encodedResult, cardinality);
+ encodePos =
+ SubcolumnFullTest.bitPacking(
+ subcolumnList[i], dictBitWidth, encodePos, encodedResult, listLength);
+ continue;
+ }
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+ encodePos =
+ SubcolumnFullTest.bitPacking(
+ subcolumnList[i], bitWidthList[i], encodePos, encodedResult, listLength);
+ } else {
+ encodingType[i] = 1;
+
+ encodedResult[encodePos] = (byte) (index >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (index & 0xFF);
+ encodePos += 1;
+
+ index = 0;
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < listLength; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ runLength[index] = j;
+ rleValues[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ runLength[index] = listLength;
+ rleValues[index] = previous;
+ index++;
+
+ encodePos = SubcolumnFullTest.bitPacking(runLength, bw, encodePos, encodedResult, index);
+ encodePos =
+ SubcolumnFullTest.bitPacking(
+ rleValues, bitWidthList[i], encodePos, encodedResult, index);
+ }
+ }
+
+ if (stats != null) {
+ stats.recordEncodingType(encodingType, l);
+ }
+
+ SubcolumnFullTest.bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int BlockEncoder(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ EncodingTypeStats stats) {
+ int[] minDelta = new int[3];
+ int[] dataDelta =
+ SubcolumnFullTest.getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ SubcolumnFullTest.int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ // Keep SubcolumnFull behavior: only pick beta on the first block.
+ if (blockIndex == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+ int m = SubcolumnFullTest.bitWidth(maxValue);
+ beta[0] = SubcolumnFullTest.Subcolumn(dataDelta, remainder, m, blockSize);
+ }
+
+ return SubcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize, stats);
+ }
+
+ public static int Encoder(
+ int[] data, int blockSize, byte[] encodedResult, EncodingTypeStats stats) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ SubcolumnFullTest.int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ SubcolumnFullTest.int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+
+ if (stats != null) {
+ stats.setTotalBlockCount(numBlocks + (remainder > 0 ? 1 : 0));
+ }
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult, beta, stats);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ SubcolumnFullTest.int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockEncoder(
+ data, numBlocks, blockSize, remainder, encodePos, encodedResult, beta, stats);
+ }
+
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(
+ byte[] encodedResult, int encodePos, int[] list, int blockSize) {
+ int listLength = list.length;
+ int m = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = SubcolumnFullTest.bitWidth(blockSize);
+ int beta = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnFullTest.decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][listLength];
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnFullTest.decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encodePos =
+ SubcolumnFullTest.decodeBitPacking(
+ encodedResult, encodePos, bitWidth, listLength, subcolumnList[i]);
+ } else if (type == 1) {
+ int index =
+ ((encodedResult[encodePos] & 0xFF) << 8) | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int[] runLength = new int[index];
+ int[] rleValues = new int[index];
+
+ encodePos = SubcolumnFullTest.decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos =
+ SubcolumnFullTest.decodeBitPacking(
+ encodedResult, encodePos, bitWidth, index, rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality =
+ ((encodedResult[encodePos] & 0xFF) << 8) | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int dictBitWidth = SubcolumnFullTest.bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ int[] dictValueList = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ dictValueList[j] = j;
+ }
+
+ encodePos =
+ SubcolumnFullTest.decodeBitPacking(
+ encodedResult, encodePos, bitWidthList[i], cardinality, dictKeyList);
+ encodePos =
+ SubcolumnFullTest.decodeBitPacking(
+ encodedResult, encodePos, dictBitWidth, listLength, subcolumnList[i]);
+
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dictValueList[j], dictKeyList[j]);
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnList[i][j] = valueToCode.get(subcolumnList[i][j]);
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < listLength; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int BlockDecoder(
+ byte[] encodedResult, int blockIndex, int blockSize, int remainder, int encodePos, int[] data) {
+ int[] minDelta = new int[3];
+ minDelta[0] = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int[] blockData = new int[remainder];
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, blockData, blockSize);
+
+ for (int i = 0; i < remainder; i++) {
+ data[blockIndex * blockSize + i] = blockData[i] + minDelta[0];
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+ int dataLength = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[numBlocks * blockSize + i] = SubcolumnFullTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos, data);
+ }
+
+ return data;
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ // String outputParentDir = parentDir + "result/";
+ String outputParentDir = "D://encoding-subcolumn/result/";
+ String outputPath = outputParentDir + "subcolumn_full_encoding_type_ratio_2048.csv";
+
+ int blockSize = 512;
+ blockSize = 2048;
+
+ int repeatTime = 20;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ "Block Count",
+ "Subcolumn Count",
+ "Bit Packing Subcolumn Count",
+ "Bit Packing Ratio",
+ "RLE Subcolumn Count",
+ "RLE Ratio",
+ "Dictionary Subcolumn Count",
+ "Dictionary Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = SubcolumnFullTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = SubcolumnFullTest.getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[data2Arr.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressedSize = 0;
+ int length = 0;
+ EncodingTypeStats stats = new EncodingTypeStats();
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ stats = new EncodingTypeStats();
+ length = Encoder(data2Arr, blockSize, encodedResult, stats);
+ }
+ long end = System.nanoTime();
+ encodeTime += ((end - start) / repeatTime);
+ compressedSize += length;
+
+ double ratioTmp = compressedSize / (double) (data1.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ end = System.nanoTime();
+ decodeTime += ((end - start) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Full)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio),
+ String.valueOf(stats.getTotalBlockCount()),
+ String.valueOf(stats.getTotalSubcolumnCount()),
+ String.valueOf(stats.getBitPackingSubcolumnCount()),
+ String.valueOf(stats.getBitPackingRatio()),
+ String.valueOf(stats.getRleSubcolumnCount()),
+ String.valueOf(stats.getRleRatio()),
+ String.valueOf(stats.getDictionarySubcolumnCount()),
+ String.valueOf(stats.getDictionaryRatio())
+ };
+ writer.writeRecord(record);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullTest.java
new file mode 100644
index 0000000..11beadb
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnFullTest.java
@@ -0,0 +1,74 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import org.apache.iotdb.tsfile.encoding.SubcolumnAblationPruneNewEngine.Mode;
+
+public class SubcolumnFullTest {
+
+ public static int bitWidth(int value) {
+ return SubcolumnPruneNewTest.bitWidth(value);
+ }
+
+ public static void intByte2Bytes(int value, int encodePos, byte[] encodedResult) {
+ SubcolumnPruneNewTest.intByte2Bytes(value, encodePos, encodedResult);
+ }
+
+ public static void int2Bytes(int value, int encodePos, byte[] encodedResult) {
+ SubcolumnPruneNewTest.int2Bytes(value, encodePos, encodedResult);
+ }
+
+ public static int bytes2Integer(byte[] encodedResult, int encodePos, int byteNum) {
+ return SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, byteNum);
+ }
+
+ public static int bitPacking(
+ int[] numbers, int bitWidth, int encodePos, byte[] encodedResult, int length) {
+ return SubcolumnPruneNewTest.bitPacking(numbers, bitWidth, encodePos, encodedResult, length);
+ }
+
+ public static int decodeBitPacking(
+ byte[] encodedResult,
+ int encodePos,
+ int bitWidth,
+ int length,
+ int[] numbers) {
+ return SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, encodePos, bitWidth, length, numbers);
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] tsBlock, int blockIndex, int blockSize, int remainder, int[] minDelta) {
+ return SubcolumnPruneNewTest.getAbsDeltaTsBlock(
+ tsBlock, blockIndex, blockSize, remainder, minDelta);
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize) {
+ return SubcolumnPruneNewTest.Subcolumn(
+ x, xLength, m, blockSize, SubcolumnPruneNewTest.borrowEncodingTypeBuffer());
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return SubcolumnAblationPruneNewEngine.Encoder(data, blockSize, encodedResult, Mode.FULL);
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ return SubcolumnAblationPruneNewEngine.Decoder(encodedResult);
+ }
+
+ public static String extractFileName(String filePath) {
+ return SubcolumnPruneNewTest.extractFileName(filePath);
+ }
+
+ public static int getDecimalPrecision(String numberStr) {
+ return SubcolumnPruneNewTest.getDecimalPrecision(numberStr);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ SubcolumnAblationPruneNewEngine.runAblationBenchmark(
+ "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_full.csv", Mode.FULL);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBPTest.java
new file mode 100644
index 0000000..54994d2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBPTest.java
@@ -0,0 +1,1002 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongBPTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnBP(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ long[][] subcolumnList = new long[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ // int rleCost = 0;
+
+ // // int count = 1;
+ // long currentNumber = subcolumnList[i][0];
+
+ // int index = 0;
+
+ // boolean bpBest = false;
+
+ // for (int j = 1; j < x_length; j++) {
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ // }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // index++;
+
+ // System.out.println("index: " + index);
+
+ // rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += bpCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ // encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ // int bpCost = bitWidthList[i] * list_length;
+ // int rleCost = 0;
+
+ // long previous = subcolumnList[i][0];
+ // int index = 0;
+
+ // for (int j = 1; j < list_length; j++) {
+ // long currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // index++;
+ // previous = currentNumber;
+ // }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ // }
+
+ // index++;
+
+ // rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ // encodingType[i] = 1;
+
+ // encoded_result[encode_pos] = (byte) (index >> 8);
+ // encode_pos += 1;
+ // encoded_result[encode_pos] = (byte) (index & 0xFF);
+ // encode_pos += 1;
+
+ // index = 0;
+ // int[] run_length = new int[list_length];
+ // long[] rle_values = new long[list_length];
+ // previous = subcolumnList[i][0];
+
+ // for (int j = 1; j < list_length; j++) {
+ // long currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // run_length[index] = j;
+ // rle_values[index] = previous;
+ // index++;
+ // previous = currentNumber;
+ // }
+ // }
+
+ // run_length[index] = list_length;
+ // rle_values[index] = previous;
+ // index++;
+
+ // encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ // encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+ }
+
+ // preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, long[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ // int[] encodingType = new int[l];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ // int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ // if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ // } else {
+ // int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ //
+ // encode_pos += 2;
+ //
+ // int[] run_length = new int[index];
+ // int[] rle_values = new int[index];
+ //
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+ //
+ // int currentIndex = 0;
+ // for (int j = 0; j < index; j++) {
+ // int endPos = run_length[j];
+ // int value = rle_values[j];
+ // while (currentIndex < endPos) {
+ // subcolumnList[i][currentIndex] = value;
+ // currentIndex++;
+ // }
+ // }
+ // }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ // beta[0] = SubcolumnBP(data_delta, remainder, m, block_size);
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("beta: " + beta[0]);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_long_bp_repeat200.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ repeatTime = 200;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ // test
+ // for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // System.out.print(data2_arr_decoded[i] + " ");
+ // }
+ // System.out.println();
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBetaTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBetaTest.java
new file mode 100644
index 0000000..6862b51
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBetaTest.java
@@ -0,0 +1,340 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnLongBetaTest {
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ encode_pos = SubcolumnLongTest.SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnLongTest.SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result, int beta_value) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = beta_value;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/compression_vs_beta/";
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+
+ int block_size = 1024;
+
+ int repeatTime = 100;
+
+ for (int beta : beta_list) {
+
+ String outputPath = output_parent_dir + "subcolumn_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println("beta: " + beta);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBlockSizeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBlockSizeTest.java
new file mode 100644
index 0000000..d851235
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongBlockSizeTest.java
@@ -0,0 +1,213 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongBlockSizeTest {
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir =
+ // "D:/encoding-subcolumn/result/compression_vs_block/";
+ String output_parent_dir = parent_dir + "result/compression_vs_block/";
+
+ File outputDir = new File(output_parent_dir);
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ }
+
+ int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ int repeatTime = 100;
+ repeatTime = 200;
+
+ // repeatTime = 1;
+
+ // 定义数据集名称列表
+ List<String> datasetList = new ArrayList<>();
+ datasetList.add("Arade4");
+ datasetList.add("Bird-migration");
+ datasetList.add("Bitcoin-price");
+ datasetList.add("City-temp");
+ datasetList.add("Dewpoint-temp");
+ datasetList.add("EPM-Education");
+ datasetList.add("Gov10");
+ datasetList.add("POI-lat");
+ datasetList.add("IR-bio-temp");
+ datasetList.add("PM10-dust");
+ datasetList.add("Stocks-DE");
+ datasetList.add("Stocks-UK");
+ datasetList.add("Stocks-USA");
+ datasetList.add("Wind-Speed");
+ datasetList.add("Wine-Tasting");
+
+ for (int block_size : block_size_list) {
+
+ String outputPath = output_parent_dir + "subcolumn_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ // 使用数据集名称列表循环
+ for (String datasetName : datasetList) {
+ String filePath = input_parent_dir + datasetName + ".csv";
+ File file = new File(filePath);
+
+ // 检查文件是否存在
+ if (!file.exists()) {
+ System.out.println("File not found: " + filePath);
+ continue;
+ }
+
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = SubcolumnLongTest.Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println("Block size: " + block_size);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D3.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D3.java
new file mode 100644
index 0000000..c40cda6
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D3.java
@@ -0,0 +1,1110 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+public class SubcolumnLongMaterializeR1D3 {
+
+ public static void QueryTwoColumns(byte[] encoded_result1, byte[] encoded_result2, long upper_bound1,
+ long upper_bound2) {
+ int[] first_column_results = new int[encoded_result1.length];
+ int[] first_result_length = new int[1];
+
+ Query(encoded_result1, upper_bound1, first_column_results, first_result_length);
+
+ long[] final_results = new long[first_result_length[0]];
+ int[] final_result_length = new int[1];
+
+ QueryWithIndices(encoded_result2, upper_bound2, first_column_results, first_result_length[0],
+ final_results, final_result_length);
+ }
+
+ public static void QueryWithIndices(byte[] encoded_result, long upper_bound,
+ int[] candidate_indices, int candidate_length,
+ long[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ | ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 初始化结果索引
+ result_length[0] = 0;
+
+ int[] blockIndicesCount = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ blockIndicesCount[blockIndex]++;
+ }
+
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int i = 0; i <= num_blocks; i++) {
+ blockIndices[i] = new int[blockIndicesCount[i]];
+ }
+
+ int[] currentIndices = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ int localIndex = index % block_size;
+
+ blockIndices[blockIndex][currentIndices[blockIndex]] = localIndex;
+ currentIndices[blockIndex]++;
+ }
+
+ // 遍历所有块
+ for (int i = 0; i < num_blocks; i++) {
+
+ if (blockIndicesCount[i] == 0) {
+ // 计算跳过此块所需的字节数
+ encode_pos = SkipBlock(encoded_result, i, block_size,
+ block_size, encode_pos);
+ continue;
+ }
+
+ // 对该块中的候选索引执行查询
+ encode_pos = BlockQueryWithIndices(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ blockIndices[i], blockIndicesCount[i], result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder > 0) {
+ if (blockIndicesCount[num_blocks] > 0) {
+ if (remainder <= 3) {
+ for (int j = 0; j < blockIndicesCount[num_blocks]; j++) {
+ int idx = blockIndices[num_blocks][j];
+ int offset = num_blocks * block_size + idx;
+ if (offset < data_length) {
+ int value = ((encoded_result[encode_pos + idx * 4] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + idx * 4 + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + idx * 4 + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + idx * 4 + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = offset;
+ result_length[0]++;
+ }
+ }
+ }
+ encode_pos += remainder * 4;
+ } else {
+
+ encode_pos = BlockQueryWithIndices(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ blockIndices[num_blocks], blockIndicesCount[num_blocks], result, result_length);
+ }
+ } else {
+ // 没有候选索引，跳过剩余部分
+ if (remainder <= 3) {
+ encode_pos += remainder * 4;
+ } else {
+ encode_pos = SkipBlock(encoded_result, num_blocks, block_size,
+ remainder, encode_pos);
+ }
+ }
+ }
+ }
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockQueryWithIndices(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] candidate_indices, int candidate_length,
+ long[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 所有索引默认都是候选索引
+ int[] filtered_indices = new int[candidate_length];
+ int filtered_length = candidate_length;
+ System.arraycopy(candidate_indices, 0, filtered_indices, 0, candidate_length);
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < filtered_length; i++) {
+ result[result_length[0]] = block_size * block_index + filtered_indices[i];
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < filtered_length; j++) {
+ int index = filtered_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ filtered_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ filtered_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ // 为每个候选索引查找对应的RLE值
+ for (int j = 0; j < filtered_length; j++) {
+ int index_candidate = filtered_indices[j];
+
+ // 查找包含此索引的RLE段
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ filtered_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ filtered_length = new_length;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ private static int SkipBlock(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // int[] min_delta = new int[3];
+
+ encode_pos += 8;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// System.out.println("beta:"+beta);
+// System.out.println("m:"+m);
+// System.out.println("m:"+(m + beta - 1) / beta);
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8;
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ encode_pos = (encode_pos * 8 + bw * index + 7) / 8;
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * index + 7) / 8;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static void Query(byte[] encoded_result, long upper_bound, int[] result, int[] result_length) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ // int[] result = new int[data_length];
+ // int[] result_length = new int[1];
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ if (value < upper_bound) {
+ result[result_length[0]] = data_length / block_size * block_size +i;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+ upper_bound -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 upper_bound 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ static class DecodedBlock {
+ long[] values;
+ int newEncodePos;
+ DecodedBlock(long[] values, int newEncodePos) { this.values = values; this.newEncodePos = newEncodePos; }
+ }
+
+ /**
+ * 读 big-endian 32-bit int（与 Query/BlockQueryIndex 中的读取一致）。
+ */
+ private static int readInt32BE(byte[] arr, int pos) {
+ return ((arr[pos] & 0xFF) << 24) | ((arr[pos + 1] & 0xFF) << 16) | ((arr[pos + 2] & 0xFF) << 8)
+ | (arr[pos + 3] & 0xFF);
+ }
+
+ /**
+ * 严格地完整解码一个块（反向实现 BlockQueryIndex/WithIndices 的解析逻辑）。
+ *
+ * @param encoded_result 整列的字节数组
+ * @param block_index 块索引（仅用于可读性/日志，函数内部不用它定位）
+ * @param block_size 标称块大小（用于计算 bw）
+ * @param remainder 本块的元素数（最后一个块可能小于 block_size）
+ * @param encode_pos 当前字节偏移（函数会从这里读取，并返回更新后的字节偏移）
+ * @return DecodedBlock，包含本块每个位置的解码整型值（长度 = remainder）以及新的 encode_pos
+ */
+ public static DecodedBlock decodeBlockValues(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // 1) 读取 min_delta
+ long min_delta =bytes2Long(encoded_result, encode_pos, 8);;
+ encode_pos += 8;
+
+ // 2) 读取 m
+ int m = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 如果 m == 0：表示无 subcolumns（所有值等于 min_delta）
+ if (m == 0) {
+ long[] vals = new long[remainder];
+ if (remainder > 0) Arrays.fill(vals, min_delta);
+ return new DecodedBlock(vals, encode_pos);
+ }
+
+ // 3) 其他元信息
+ int bw = SubcolumnTest.bitWidth(block_size); // 基本宽度，用于 RLE run-length 的位宽
+ int beta = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ // 4) 读取 bitWidthList（每个 subcolumn 的位宽，使用 decodeBitPacking(bits=8)）
+ int[] bitWidthList = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // 5) 读取 encodingType（每个 subcolumn 的编码类型：0=bitpacked, 1=RLE）
+ int[] encodingType = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ // 6) 逐个 subcolumn（从高位 i=l-1 到 i=0）合成值
+ long[] blockValues = new long[remainder];
+ Arrays.fill(blockValues, 0);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bw_i = bitWidthList[i];
+
+ if (type == 0) {
+ // bitpacked: 在当前 encode_pos (字节偏移) 的位流上连续存 remainder 个 bw_i 位值
+ int bitStart = encode_pos * 8; // 转为位偏移
+ // 对每个位置抽取 bw_i 位并左移累加
+ for (int p = 0; p < remainder; p++) {
+ int bitOffset = bitStart + p * bw_i;
+ int part = SubcolumnTest.bytesToInt(encoded_result, bitOffset, bw_i);
+ blockValues[p] |= (part << (i * beta));
+ }
+ // 跳过这段 bitpacked 的位段，回到下一个字节边界
+ encode_pos = (bitStart + remainder * bw_i + 7) / 8;
+ } else {
+ // RLE: 先读 runCount (2 bytes)，随后是 run_length[]（bw 位）和 rle_values[]（bw_i 位）
+ int runCount = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ // 读取 run_length（每项 bw 位）
+ int[] run_length = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, runCount, run_length);
+
+ // 读取 rle_values（每项 bw_i 位）
+ int[] rle_values = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw_i, runCount, rle_values);
+
+ // 根据 run_lengths 展开并赋值
+ int pos = 0;
+ for (int r = 0; r < runCount && pos < remainder; r++) {
+ int len = run_length[r];
+ int val = rle_values[r];
+ for (int t = 0; t < len && pos < remainder; t++, pos++) {
+ blockValues[pos] |= (val << (i * beta));
+ }
+ }
+ // 注意：encode_pos 已由 decodeBitPacking 更新
+ }
+ }
+
+ // 7) 把 min_delta 加回每个位置
+ for (int p = 0; p < remainder; p++) {
+ blockValues[p] += min_delta;
+ }
+
+ return new DecodedBlock(blockValues, encode_pos);
+ }
+
+ /**
+ * 完整解码整列（逐块调用 decodeBlockValues）
+ *
+ * @param encoded_result 编码后字节数组（包含前 8 字节 header: data_length, block_size）
+ * @return 解码后的整列 int[]，长度 = data_length
+ */
+ public static long[] decodeColumnFully(byte[] encoded_result) {
+ int encode_pos = 0;
+ int data_length = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+ int block_size = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+ long[] out = new long[data_length];
+
+ // 完整块
+ for (int b = 0; b < num_blocks; b++) {
+ DecodedBlock db = decodeBlockValues(encoded_result, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, b * block_size, block_size);
+ }
+
+ // 最后一块（如果有剩余）
+ if (remainder > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result, num_blocks, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, num_blocks * block_size, remainder);
+ }
+
+ return out;
+ }
+
+ /**
+ * 严格 EM-parallel: 彻底解码两列为 int[]，然后逐行判断两个谓词同时成立的位置。
+ *
+ * @param encoded_result1 列1的编码字节数组
+ * @param encoded_result2 列2的编码字节数组
+ * @param upper_bound1 列1的上界谓词（< upper_bound1）
+ * @param upper_bound2 列2的上界谓词（< upper_bound2）
+ * @param result 用于输出匹配位置的数组（全表偏移 / 行号）
+ * @param result_length 长度容器（长度为1的数组，写回匹配数）
+ */
+ public static void QueryTwoColumnsStrictEMParallel(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, long[] result, int[] result_length) {
+
+ // 完整解码两列
+ long[] col1 = decodeColumnFully(encoded_result1);
+ long[] col2 = decodeColumnFully(encoded_result2);
+
+ // 确定行数（取两列最小）
+ int n = Math.min(col1.length, col2.length);
+ result_length[0] = 0;
+
+ for (int i = 0; i < n; i++) {
+ if (col1[i] < upper_bound1 && col2[i] < upper_bound2) {
+ result[result_length[0]] = i;
+ result_length[0]++;
+ }
+ }
+ }
+
+ /**
+ * 严格 EM-pipelined: 完整解码第一列（物化为 values），根据第一列筛出 candidate positions（按块组织），
+ * 然后对第二列按块按需解码（只解包含 candidate 的块），在这些块中对 candidate 的局部索引做精确判定。
+ *
+ * @param encoded_result1 列1编码
+ * @param encoded_result2 列2编码
+ * @param upper_bound1
+ * @param upper_bound2
+ * @param result
+ * @param result_length
+ */
+ public static void QueryTwoColumnsStrictEMPipelined(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, long[] result, int[] result_length) {
+
+ // 1) 解码第 1 列（完整解码以物化 tuple 的该属性）
+ long[] col1 = decodeColumnFully(encoded_result1);
+ int data_length = col1.length;
+
+ // 2) 构建候选位置分块索引（与 QueryWithIndices 中的策略一致）
+ int block_size = readInt32BE(encoded_result2, 4); // 注意：编码头部：前4字节 data_length，接着 4 字节 block_size
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 统计每个块中 candidate 数量
+ int[] blockCount = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ blockCount[bidx]++;
+ }
+ }
+
+ // 若没有 candidate，快速返回
+ int totalCandidates = 0;
+ for (int c : blockCount) totalCandidates += c;
+ result_length[0] = 0;
+ if (totalCandidates == 0) return;
+
+ // 为每块分配数组以记录 local indices
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int b = 0; b <= num_blocks; b++) {
+ blockIndices[b] = new int[blockCount[b]];
+ }
+ int[] cursor = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ int local = i % block_size;
+ blockIndices[bidx][cursor[bidx]++] = local;
+ }
+ }
+
+ // 3) 逐块扫描第二列：若该块没有 candidate -> 跳过（SkipBlock）；否则解码该块并测试
+ int encode_pos = 0;
+ int data_len_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ int bs_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ // sanity check bs_from_header == block_size
+ // 逐块循环
+ for (int b = 0; b < num_blocks; b++) {
+ if (blockCount[b] == 0) {
+ // 跳过此块
+ encode_pos = SkipBlock(encoded_result2, b, block_size, block_size, encode_pos);
+ continue;
+ }
+ // 需要解码此块
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ long[] vals = db.values;
+ // 对该块的候选局部索引测试第二列谓词
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ }
+
+ // 最后可能的不完整块
+ if (remainder > 0) {
+ int b = num_blocks;
+ if (blockCount[b] > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ long[] vals = db.values;
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ } else {
+ // 无 candidate，跳过或不处理
+ }
+ }
+ }
+
+ public static double computeSelectivity(long len1_0, long len2_0, long halfSize, long match) {
+ double sA = Double.NaN, sB = Double.NaN, pAB = Double.NaN, lift = Double.NaN;
+
+ if (halfSize <= 0) return lift;
+
+ // 基本概率
+ sA = (double) len1_0 / (double) halfSize;
+ sB = (double) len2_0 / (double) halfSize;
+ pAB = (double) match / (double) halfSize;
+
+ // lift = P(A∧B) / (P(A) P(B))，仅在分母非零时计算
+ if (sA > 0.0 && sB > 0.0) {
+ lift = pAB / (sA * sB);
+ }
+
+ return lift;
+ }
+ public static double phiCoefficient(long len1_0, long len2_0, long halfSize, long match) {
+ // 2x2 表格元素
+ double a = (double) match; // A ∧ B
+ double b = (double) (len1_0 - match); // A ∧ ¬B
+ double c = (double) (len2_0 - match); // ¬A ∧ B
+ double d = (double) (halfSize - (match + (len1_0 - match) + (len2_0 - match)));
+ // 等价于: d = halfSize - (a + b + c)
+
+ // 如果任何分量为负，输入可能不合法，返回 NaN
+ if (a < 0 || b < 0 || c < 0 || d < 0) {
+ return Double.NaN;
+ }
+
+ double numerator = a * d - b * c;
+ double denomTerm1 = (a + b) * (c + d);
+ double denomTerm2 = (a + c) * (b + d);
+
+ // 分母为 sqrt( denomTerm1 * denomTerm2 )
+ double denomProduct = denomTerm1 * denomTerm2;
+ if (denomProduct <= 0.0) {
+ return Double.NaN; // 避免除零或根号负数
+ }
+
+ double phi = numerator / Math.sqrt(denomProduct);
+ return phi;
+ }
+ // 放在类的末尾，作为一个新的测试 / helper
+ @Test
+ public void compareMaterializationStrategies() throws IOException {
+ // --- 基本设置，复用你 testQuery 中的路径 / 数据准备逻辑 ---
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ // String output_parent_dir = parent_dir + "result/materialization/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/materialization/";
+
+// // 这里为了演示，仅处理单个 CSV 文件（你可以循环多个文件）
+// File directory = new File(input_parent_dir);
+// File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+// if (csvFiles == null || csvFiles.length == 0) {
+// System.out.println("No csv files found under " + input_parent_dir);
+// return;
+// }
+//
+// // 选第一个文件作为 demo
+// File file = csvFiles[0];
+// String datasetName = extractFileName(file.toString());
+// System.out.println("Dataset: " + datasetName);
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 75000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 500;
+ String outputPath = output_parent_dir + "subcolumn_materialization.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "LM-pipelined",
+ "LM-parallel",
+ "EM-pipelined",
+ "EM-parallel",
+ "Points",
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // 读取列并构造两列（复用你已有的读取逻辑）
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(!queryRange.containsKey(datasetName))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> raw = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ raw.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = raw.size();
+ int halfSize = totalSize / 2;
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) col1_data[i] = (long) (raw.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++) col2_data[i] = (long) (raw.get(i + halfSize) * max_mul);
+
+// if(datasetName.equals("Stocks-UK")){
+// System.out.println(Arrays.toString(col1_data));
+// }
+ int block_size = 512; // 选择一个 block size 做比较
+// int repeatTime = 200;
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ // 编码（复用你的 Encoder）
+ int length1 = SubcolumnLongTest.Encoder(col1_data, block_size, encoded_result1);
+ int length2 = SubcolumnLongTest.Encoder(col2_data, block_size, encoded_result2);
+
+ int upper = queryRange.containsKey(datasetName) ? queryRange.get(datasetName) : Integer.MAX_VALUE;
+
+// System.out.println("Running repeats: " + repeatTime + " upper=" + upper);
+
+ // ---------- 1) LM-pipelined: your existing QueryTwoColumns ----------
+ long tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ QueryTwoColumns(encoded_result1, encoded_result2, upper, upper);
+ }
+ long tEnd = System.nanoTime();
+ long lmPipelinedTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-pipelined avg ns: " + lmPipelinedTime);
+
+ // ---------- 2) LM-parallel: Query both columns separately -> intersect positions ----------
+ // helper arrays reused
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ // warm run to avoid JIT one-time overhead bias
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ double selectivity = 0;
+ double phi = 0;
+ int match = 0;
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ CompletableFuture<Void> future1 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result1, upper, res1, len1);
+ });
+
+ CompletableFuture<Void> future2 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result2, upper, res2, len2);
+ });
+
+ // 等待两个查询都完成
+ try {
+ CompletableFuture.allOf(future1, future2).get();
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ // 处理异常，可能需要中断循环或采取其他措施
+ Thread.currentThread().interrupt(); // 重新设置中断状态
+ break;
+ }
+ // intersect result sets (they are arrays of positions)
+//System.out.println(len1[0]);
+// System.out.println(len2[0]);
+//// 并行设置bit
+ long[] bits1 = new long[(halfSize + 63) / 64];
+ long[] bits2 = new long[(halfSize + 63) / 64];
+
+// 设置bit
+ for (int i = 0; i < len1[0]; i++) {
+ int pos = res1[i];
+ bits1[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ for (int i = 0; i < len2[0]; i++) {
+ int pos = res2[i];
+ bits2[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+// 求交集并计数
+ match = 0;
+ for (int i = 0; i < bits1.length; i++) {
+ long intersection = bits1[i] & bits2[i];
+ match += Long.bitCount(intersection);
+ }
+ }
+ tEnd = System.nanoTime();
+// selectivity = computeSelectivity(len1[0],len2[0],halfSize,match);
+// phi = phiCoefficient(len1[0],len2[0],halfSize,match);
+// System.out.println(len1[0]+","+len2[0]);
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+
+ long[] result = new long[encoded_result1.length];
+ int[] resultLen = new int[1];
+
+// ---------- 3) EM-pipelined (近似实现说明) ----------
+ // 说明：严格的 EM-pipelined 需要把第一列物化成 (pos,value) tuples（即完整解码得到值），
+ // 然后按这些 pos 去第二列延展并筛选。要做到严格，需实现「按索引只解码第二列对应位置的值」或把第二列解码成数组。
+ // 在这里给出一个“可运行的近似实现”：把第一列先用 Query() 拿到候选 positions（pos），
+ // 再把这些 pos 作为 candidate 传入 QueryWithIndices(encoded_result2,...)
+ // （注意：这是 LM-pipelined 与 EM-pipelined 在含义上并非完全相同，但在当前可用接口下这是能运行且可比较的实现）
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // 获取第一列候选位置（认为已“物化”成pos list）
+ resultLen[0] = 0;
+ QueryTwoColumnsStrictEMPipelined(encoded_result1, encoded_result2, queryRange.get(datasetName), queryRange.get(datasetName), result, resultLen);
+
+// Query(encoded_result1, upper, res1, len1);
+// // 用这些位置去查询第二列（QueryWithIndices 将仅在这些位置上做判断）
+// QueryWithIndices(encoded_result2, upper, res1, len1[0], res2, len2);
+// // res2 中现在是满足第二列 < upper 的偏移（相对于全列偏移），如果需要对第一列的值也做判定，需要把第一列解码成values，这里省略
+ }
+ tEnd = System.nanoTime();
+ long emPipelinedApproxTime = (tEnd - tStart) / repeatTime;
+// System.out.println("EM-pipelined strict matched: " + resultLen[0]);
+ System.out.println("EM-pipelined (approx) avg ns: " + emPipelinedApproxTime);
+
+ // ---------- 4) EM-parallel ----------
+
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // 近似实现：同时对两列调用 QueryWithIndices（先对第一列获取候选pos，然后把这些pos作为输入去第二列）
+ QueryTwoColumnsStrictEMParallel(encoded_result1, encoded_result2, queryRange.get(datasetName), queryRange.get(datasetName), result, resultLen);
+// Query(encoded_result1, upper, res1, len1); // first column candidates
+// QueryWithIndices(encoded_result2, upper, res1, len1[0], res2, len2);
+ // 结果 res2 表示在那些 pos 中满足第二列条件的偏移
+ }
+ tEnd = System.nanoTime();
+// System.out.println("EM-parallel strict matched: " + resultLen[0]);
+ long emParallelApproxTime = (tEnd - tStart) / repeatTime;
+ System.out.println("EM-parallel (approx) avg ns: " + emParallelApproxTime);
+
+ // 最后打印一行小结
+ System.out.println("Summary (ns avg per query): LM-pipelined=" + lmPipelinedTime
+ + " LM-parallel=" + lmParallelTime
+ + " EM-pipelined-approx=" + emPipelinedApproxTime
+ + " EM-parallel-approx=" + emParallelApproxTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(lmPipelinedTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(emPipelinedApproxTime),
+ String.valueOf(emParallelApproxTime),
+ String.valueOf(totalSize)
+ };
+
+
+ writer.writeRecord(record);
+
+ }
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D8.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D8.java
new file mode 100644
index 0000000..059b0de
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongMaterializeR1D8.java
@@ -0,0 +1,1068 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+public class SubcolumnLongMaterializeR1D8 {
+
+ public static void QueryTwoColumns(byte[] encoded_result1, byte[] encoded_result2, long upper_bound1,
+ long upper_bound2) {
+ int[] first_column_results = new int[encoded_result1.length];
+ int[] first_result_length = new int[1];
+
+ Query(encoded_result1, upper_bound1, first_column_results, first_result_length);
+
+ long[] final_results = new long[first_result_length[0]];
+ int[] final_result_length = new int[1];
+
+ QueryWithIndices(encoded_result2, upper_bound2, first_column_results, first_result_length[0],
+ final_results, final_result_length);
+ }
+
+ public static void QueryWithIndices(byte[] encoded_result, long upper_bound,
+ int[] candidate_indices, int candidate_length,
+ long[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ | ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 初始化结果索引
+ result_length[0] = 0;
+
+ int[] blockIndicesCount = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ blockIndicesCount[blockIndex]++;
+ }
+
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int i = 0; i <= num_blocks; i++) {
+ blockIndices[i] = new int[blockIndicesCount[i]];
+ }
+
+ int[] currentIndices = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ int localIndex = index % block_size;
+
+ blockIndices[blockIndex][currentIndices[blockIndex]] = localIndex;
+ currentIndices[blockIndex]++;
+ }
+
+ // 遍历所有块
+ for (int i = 0; i < num_blocks; i++) {
+
+ if (blockIndicesCount[i] == 0) {
+ // 计算跳过此块所需的字节数
+ encode_pos = SkipBlock(encoded_result, i, block_size,
+ block_size, encode_pos);
+ continue;
+ }
+
+ // 对该块中的候选索引执行查询
+ encode_pos = BlockQueryWithIndices(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ blockIndices[i], blockIndicesCount[i], result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder > 0) {
+ if (blockIndicesCount[num_blocks] > 0) {
+ if (remainder <= 3) {
+ for (int j = 0; j < blockIndicesCount[num_blocks]; j++) {
+ int idx = blockIndices[num_blocks][j];
+ int offset = num_blocks * block_size + idx;
+ if (offset < data_length) {
+ int value = ((encoded_result[encode_pos + idx * 4] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + idx * 4 + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + idx * 4 + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + idx * 4 + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = offset;
+ result_length[0]++;
+ }
+ }
+ }
+ encode_pos += remainder * 4;
+ } else {
+
+ encode_pos = BlockQueryWithIndices(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ blockIndices[num_blocks], blockIndicesCount[num_blocks], result, result_length);
+ }
+ } else {
+ // 没有候选索引，跳过剩余部分
+ if (remainder <= 3) {
+ encode_pos += remainder * 4;
+ } else {
+ encode_pos = SkipBlock(encoded_result, num_blocks, block_size,
+ remainder, encode_pos);
+ }
+ }
+ }
+ }
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockQueryWithIndices(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] candidate_indices, int candidate_length,
+ long[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 所有索引默认都是候选索引
+ int[] filtered_indices = new int[candidate_length];
+ int filtered_length = candidate_length;
+ System.arraycopy(candidate_indices, 0, filtered_indices, 0, candidate_length);
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < filtered_length; i++) {
+ result[result_length[0]] = block_size * block_index + filtered_indices[i];
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < filtered_length; j++) {
+ int index = filtered_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ filtered_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ filtered_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ // 为每个候选索引查找对应的RLE值
+ for (int j = 0; j < filtered_length; j++) {
+ int index_candidate = filtered_indices[j];
+
+ // 查找包含此索引的RLE段
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ filtered_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ filtered_length = new_length;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ private static int SkipBlock(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // int[] min_delta = new int[3];
+
+ encode_pos += 8;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// System.out.println("beta:"+beta);
+// System.out.println("m:"+m);
+// System.out.println("m:"+(m + beta - 1) / beta);
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8;
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ encode_pos = (encode_pos * 8 + bw * index + 7) / 8;
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * index + 7) / 8;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static void Query(byte[] encoded_result, long upper_bound, int[] result, int[] result_length) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ // int[] result = new int[data_length];
+ // int[] result_length = new int[1];
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ if (value < upper_bound) {
+ result[result_length[0]] = data_length / block_size * block_size +i;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long upper_bound, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+ upper_bound -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 upper_bound 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ long value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ static class DecodedBlock {
+ int[] values;
+ int newEncodePos;
+ DecodedBlock(int[] values, int newEncodePos) { this.values = values; this.newEncodePos = newEncodePos; }
+ }
+
+ /**
+ * 读 big-endian 32-bit int（与 Query/BlockQueryIndex 中的读取一致）。
+ */
+ private static int readInt32BE(byte[] arr, int pos) {
+ return ((arr[pos] & 0xFF) << 24) | ((arr[pos + 1] & 0xFF) << 16) | ((arr[pos + 2] & 0xFF) << 8)
+ | (arr[pos + 3] & 0xFF);
+ }
+
+ /**
+ * 严格地完整解码一个块（反向实现 BlockQueryIndex/WithIndices 的解析逻辑）。
+ *
+ * @param encoded_result 整列的字节数组
+ * @param block_index 块索引（仅用于可读性/日志，函数内部不用它定位）
+ * @param block_size 标称块大小（用于计算 bw）
+ * @param remainder 本块的元素数（最后一个块可能小于 block_size）
+ * @param encode_pos 当前字节偏移（函数会从这里读取，并返回更新后的字节偏移）
+ * @return DecodedBlock，包含本块每个位置的解码整型值（长度 = remainder）以及新的 encode_pos
+ */
+ public static DecodedBlock decodeBlockValues(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // 1) 读取 min_delta
+ int min_delta = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ // 2) 读取 m
+ int m = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 如果 m == 0：表示无 subcolumns（所有值等于 min_delta）
+ if (m == 0) {
+ int[] vals = new int[remainder];
+ if (remainder > 0) Arrays.fill(vals, min_delta);
+ return new DecodedBlock(vals, encode_pos);
+ }
+
+ // 3) 其他元信息
+ int bw = SubcolumnTest.bitWidth(block_size); // 基本宽度，用于 RLE run-length 的位宽
+ int beta = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ // 4) 读取 bitWidthList（每个 subcolumn 的位宽，使用 decodeBitPacking(bits=8)）
+ int[] bitWidthList = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // 5) 读取 encodingType（每个 subcolumn 的编码类型：0=bitpacked, 1=RLE）
+ int[] encodingType = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ // 6) 逐个 subcolumn（从高位 i=l-1 到 i=0）合成值
+ int[] blockValues = new int[remainder];
+ Arrays.fill(blockValues, 0);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bw_i = bitWidthList[i];
+
+ if (type == 0) {
+ // bitpacked: 在当前 encode_pos (字节偏移) 的位流上连续存 remainder 个 bw_i 位值
+ int bitStart = encode_pos * 8; // 转为位偏移
+ // 对每个位置抽取 bw_i 位并左移累加
+ for (int p = 0; p < remainder; p++) {
+ int bitOffset = bitStart + p * bw_i;
+ int part = SubcolumnTest.bytesToInt(encoded_result, bitOffset, bw_i);
+ blockValues[p] |= (part << (i * beta));
+ }
+ // 跳过这段 bitpacked 的位段，回到下一个字节边界
+ encode_pos = (bitStart + remainder * bw_i + 7) / 8;
+ } else {
+ // RLE: 先读 runCount (2 bytes)，随后是 run_length[]（bw 位）和 rle_values[]（bw_i 位）
+ int runCount = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ // 读取 run_length（每项 bw 位）
+ int[] run_length = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, runCount, run_length);
+
+ // 读取 rle_values（每项 bw_i 位）
+ int[] rle_values = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw_i, runCount, rle_values);
+
+ // 根据 run_lengths 展开并赋值
+ int pos = 0;
+ for (int r = 0; r < runCount && pos < remainder; r++) {
+ int len = run_length[r];
+ int val = rle_values[r];
+ for (int t = 0; t < len && pos < remainder; t++, pos++) {
+ blockValues[pos] |= (val << (i * beta));
+ }
+ }
+ // 注意：encode_pos 已由 decodeBitPacking 更新
+ }
+ }
+
+ // 7) 把 min_delta 加回每个位置
+ for (int p = 0; p < remainder; p++) {
+ blockValues[p] += min_delta;
+ }
+
+ return new DecodedBlock(blockValues, encode_pos);
+ }
+
+ /**
+ * 完整解码整列（逐块调用 decodeBlockValues）
+ *
+ * @param encoded_result 编码后字节数组（包含前 8 字节 header: data_length, block_size）
+ * @return 解码后的整列 int[]，长度 = data_length
+ */
+ public static int[] decodeColumnFully(byte[] encoded_result) {
+ int encode_pos = 0;
+ int data_length = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+ int block_size = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+ int[] out = new int[data_length];
+
+ // 完整块
+ for (int b = 0; b < num_blocks; b++) {
+ DecodedBlock db = decodeBlockValues(encoded_result, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, b * block_size, block_size);
+ }
+
+ // 最后一块（如果有剩余）
+ if (remainder > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result, num_blocks, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, num_blocks * block_size, remainder);
+ }
+
+ return out;
+ }
+
+ /**
+ * 严格 EM-parallel: 彻底解码两列为 int[]，然后逐行判断两个谓词同时成立的位置。
+ *
+ * @param encoded_result1 列1的编码字节数组
+ * @param encoded_result2 列2的编码字节数组
+ * @param upper_bound1 列1的上界谓词（< upper_bound1）
+ * @param upper_bound2 列2的上界谓词（< upper_bound2）
+ * @param result 用于输出匹配位置的数组（全表偏移 / 行号）
+ * @param result_length 长度容器（长度为1的数组，写回匹配数）
+ */
+ public static void QueryTwoColumnsStrictEMParallel(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 完整解码两列
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int[] col2 = decodeColumnFully(encoded_result2);
+
+ // 确定行数（取两列最小）
+ int n = Math.min(col1.length, col2.length);
+ result_length[0] = 0;
+
+ for (int i = 0; i < n; i++) {
+ if (col1[i] < upper_bound1 && col2[i] < upper_bound2) {
+ result[result_length[0]] = i;
+ result_length[0]++;
+ }
+ }
+ }
+
+ /**
+ * 严格 EM-pipelined: 完整解码第一列（物化为 values），根据第一列筛出 candidate positions（按块组织），
+ * 然后对第二列按块按需解码（只解包含 candidate 的块），在这些块中对 candidate 的局部索引做精确判定。
+ *
+ * @param encoded_result1 列1编码
+ * @param encoded_result2 列2编码
+ * @param upper_bound1
+ * @param upper_bound2
+ * @param result
+ * @param result_length
+ */
+ public static void QueryTwoColumnsStrictEMPipelined(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 1) 解码第 1 列（完整解码以物化 tuple 的该属性）
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int data_length = col1.length;
+
+ // 2) 构建候选位置分块索引（与 QueryWithIndices 中的策略一致）
+ int block_size = readInt32BE(encoded_result2, 4); // 注意：编码头部：前4字节 data_length，接着 4 字节 block_size
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 统计每个块中 candidate 数量
+ int[] blockCount = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ blockCount[bidx]++;
+ }
+ }
+
+ // 若没有 candidate，快速返回
+ int totalCandidates = 0;
+ for (int c : blockCount) totalCandidates += c;
+ result_length[0] = 0;
+ if (totalCandidates == 0) return;
+
+ // 为每块分配数组以记录 local indices
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int b = 0; b <= num_blocks; b++) {
+ blockIndices[b] = new int[blockCount[b]];
+ }
+ int[] cursor = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ int local = i % block_size;
+ blockIndices[bidx][cursor[bidx]++] = local;
+ }
+ }
+
+ // 3) 逐块扫描第二列：若该块没有 candidate -> 跳过（SkipBlock）；否则解码该块并测试
+ int encode_pos = 0;
+ int data_len_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ int bs_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ // sanity check bs_from_header == block_size
+ // 逐块循环
+ for (int b = 0; b < num_blocks; b++) {
+ if (blockCount[b] == 0) {
+ // 跳过此块
+ encode_pos = SkipBlock(encoded_result2, b, block_size, block_size, encode_pos);
+ continue;
+ }
+ // 需要解码此块
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ // 对该块的候选局部索引测试第二列谓词
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ }
+
+ // 最后可能的不完整块
+ if (remainder > 0) {
+ int b = num_blocks;
+ if (blockCount[b] > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ } else {
+ // 无 candidate，跳过或不处理
+ }
+ }
+ }
+
+ public static double computeSelectivity(long len1_0, long len2_0, long halfSize, long match) {
+ double sA = Double.NaN, sB = Double.NaN, pAB = Double.NaN, lift = Double.NaN;
+
+ if (halfSize <= 0) return lift;
+
+ // 基本概率
+ sA = (double) len1_0 / (double) halfSize;
+ sB = (double) len2_0 / (double) halfSize;
+ pAB = (double) match / (double) halfSize;
+
+ // lift = P(A∧B) / (P(A) P(B))，仅在分母非零时计算
+ if (sA > 0.0 && sB > 0.0) {
+ lift = pAB / (sA * sB);
+ }
+
+ return lift;
+ }
+ public static double phiCoefficient(long len1_0, long len2_0, long halfSize, long match) {
+ // 2x2 表格元素
+ double a = (double) match; // A ∧ B
+ double b = (double) (len1_0 - match); // A ∧ ¬B
+ double c = (double) (len2_0 - match); // ¬A ∧ B
+ double d = (double) (halfSize - (match + (len1_0 - match) + (len2_0 - match)));
+ // 等价于: d = halfSize - (a + b + c)
+
+ // 如果任何分量为负，输入可能不合法，返回 NaN
+ if (a < 0 || b < 0 || c < 0 || d < 0) {
+ return Double.NaN;
+ }
+
+ double numerator = a * d - b * c;
+ double denomTerm1 = (a + b) * (c + d);
+ double denomTerm2 = (a + c) * (b + d);
+
+ // 分母为 sqrt( denomTerm1 * denomTerm2 )
+ double denomProduct = denomTerm1 * denomTerm2;
+ if (denomProduct <= 0.0) {
+ return Double.NaN; // 避免除零或根号负数
+ }
+
+ double phi = numerator / Math.sqrt(denomProduct);
+ return phi;
+ }
+ // 放在类的末尾，作为一个新的测试 / helper
+ @Test
+ public void compareMaterializationStrategies() throws IOException {
+ // --- 基本设置，复用你 testQuery 中的路径 / 数据准备逻辑 ---
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+
+// // 这里为了演示，仅处理单个 CSV 文件（你可以循环多个文件）
+// File directory = new File(input_parent_dir);
+// File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+// if (csvFiles == null || csvFiles.length == 0) {
+// System.out.println("No csv files found under " + input_parent_dir);
+// return;
+// }
+//
+// // 选第一个文件作为 demo
+// File file = csvFiles[0];
+// String datasetName = extractFileName(file.toString());
+// System.out.println("Dataset: " + datasetName);
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 75000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 500;
+ String outputPath = output_parent_dir + "subcolumn_filter.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "LM-pipelined",
+ "LM-parallel",
+ "Selectivity",
+ "Phi",
+ "Points",
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // 读取列并构造两列（复用你已有的读取逻辑）
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(!queryRange.containsKey(datasetName))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> raw = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ raw.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = raw.size();
+ int halfSize = totalSize / 2;
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) col1_data[i] = (long) (raw.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++) col2_data[i] = (long) (raw.get(i + halfSize) * max_mul);
+
+// if(datasetName.equals("Stocks-UK")){
+// System.out.println(Arrays.toString(col1_data));
+// }
+ int block_size = 512; // 选择一个 block size 做比较
+// int repeatTime = 200;
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ // 编码（复用你的 Encoder）
+ int length1 = SubcolumnLongTest.Encoder(col1_data, block_size, encoded_result1);
+ int length2 = SubcolumnLongTest.Encoder(col2_data, block_size, encoded_result2);
+
+ int upper = queryRange.containsKey(datasetName) ? queryRange.get(datasetName) : Integer.MAX_VALUE;
+
+// System.out.println("Running repeats: " + repeatTime + " upper=" + upper);
+
+ // ---------- 1) LM-pipelined: your existing QueryTwoColumns ----------
+ long tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ QueryTwoColumns(encoded_result1, encoded_result2, upper, upper);
+ }
+ long tEnd = System.nanoTime();
+ long lmPipelinedTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-pipelined avg ns: " + lmPipelinedTime);
+
+ // ---------- 2) LM-parallel: Query both columns separately -> intersect positions ----------
+ // helper arrays reused
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ // warm run to avoid JIT one-time overhead bias
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ double selectivity = 0;
+ double phi = 0;
+ int match = 0;
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ CompletableFuture<Void> future1 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result1, upper, res1, len1);
+ });
+
+ CompletableFuture<Void> future2 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result2, upper, res2, len2);
+ });
+
+ // 等待两个查询都完成
+ try {
+ CompletableFuture.allOf(future1, future2).get();
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ // 处理异常，可能需要中断循环或采取其他措施
+ Thread.currentThread().interrupt(); // 重新设置中断状态
+ break;
+ }
+ // intersect result sets (they are arrays of positions)
+//System.out.println(len1[0]);
+// System.out.println(len2[0]);
+//// 并行设置bit
+ long[] bits1 = new long[(halfSize + 63) / 64];
+ long[] bits2 = new long[(halfSize + 63) / 64];
+
+// 设置bit
+ for (int i = 0; i < len1[0]; i++) {
+ int pos = res1[i];
+ bits1[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+ for (int i = 0; i < len2[0]; i++) {
+ int pos = res2[i];
+ bits2[pos >> 6] |= (1L << (pos & 0x3F));
+ }
+
+// 求交集并计数
+ match = 0;
+ for (int i = 0; i < bits1.length; i++) {
+ long intersection = bits1[i] & bits2[i];
+ match += Long.bitCount(intersection);
+ }
+ }
+ tEnd = System.nanoTime();
+ selectivity = computeSelectivity(len1[0],len2[0],halfSize,match);
+ phi = phiCoefficient(len1[0],len2[0],halfSize,match);
+ System.out.println(len1[0]+","+len2[0]);
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+
+ int[] result = new int[encoded_result1.length];
+ int[] resultLen = new int[1];
+
+
+
+ // 最后打印一行小结
+// System.out.println("Summary (ns avg per query): LM-pipelined=" + lmPipelinedTime
+// + " LM-parallel=" + lmParallelTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(lmPipelinedTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(selectivity),
+ String.valueOf(phi),
+ String.valueOf(totalSize)
+ };
+ writer.writeRecord(record);
+
+ }
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongOnSortedTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongOnSortedTest.java
new file mode 100644
index 0000000..593a52b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongOnSortedTest.java
@@ -0,0 +1,1076 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongOnSortedTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13 };
+ // int[] beta_list = { 2, 3, 4, 7, 11 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta=1 ;beta < 5 ;beta ++ ) {
+// for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ long[][] subcolumnList = new long[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ long currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ long previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ long[] rle_values = new long[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, long[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+// if (block_index % 2 == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("beta: " + beta[0]);
+// }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/"; //""D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_long_on_sorted.csv";
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Points",
+ "Encoding Time",
+ "Decoding Time",
+ "Compressed Size",
+ "Compression Ratio",
+ "Encoding Time Sort",
+ "Decoding Time Sort",
+ "Compressed Size Sort",
+ "Compression Ratio Sort"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data1_arr = new long[data1.size()];
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ data1_arr[i] = i;
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ byte[] encoded_result1 = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int length1 = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(data1_arr, block_size, encoded_result1);
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ length += length1;
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES *2);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data1_arr_decoded = new long[data2_arr.length];
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data1_arr_decoded = Decoder(encoded_result1);
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ Integer[] indices = new Integer[data1_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = i;
+ }
+
+// 根据 data2_arr 的值对索引数组进行排序
+ long[] finalData2_arr = data2_arr;
+ Arrays.sort(indices, (i, j) -> Long.compare(finalData2_arr[i], finalData2_arr[j]));
+
+// 根据排序后的索引重新排列两个数组
+ long[] sortedData1 = new long[data1_arr.length];
+ long[] sortedData2 = new long[data2_arr.length];
+ for (int i = 0; i < indices.length; i++) {
+ sortedData1[i] = data1_arr[indices[i]];
+ sortedData2[i] = data2_arr[indices[i]];
+ }
+
+//// 将排序后的数组赋回原数组（可选）
+// data1_arr = sortedData1;
+// data2_arr = sortedData2;
+
+ System.out.println(max_decimal);
+ encoded_result = new byte[data2_arr.length * 8];
+ encoded_result1 = new byte[data2_arr.length * 8];
+
+ long encodeTime_sort = 0;
+ long decodeTime_sort = 0;
+ double ratio_sort = 0;
+ double compressed_size_sort = 0;
+
+ int length_sort = 0;
+ int length1_sort = 0;
+
+ long s_sort = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = Encoder(sortedData1, block_size, encoded_result1);
+ length = Encoder(sortedData2, block_size, encoded_result);
+ }
+
+ long e_sort = System.nanoTime();
+ encodeTime_sort += ((e_sort - s_sort) / repeatTime);
+ length += length1;
+ compressed_size_sort += length;
+
+ double ratioTmp_sort;
+
+ ratioTmp_sort = compressed_size_sort / (double) (data1.size() * Long.BYTES*2);
+
+ ratio_sort += ratioTmp_sort;
+
+ System.out.println("Decode");
+
+ long[] data1_arr_decoded_sort = new long[data2_arr.length];
+ long[] data2_arr_decoded_sort = new long[data2_arr.length];
+
+ s_sort = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data1_arr_decoded_sort = Decoder(encoded_result1);
+ data2_arr_decoded_sort = Decoder(encoded_result);
+ }
+
+ e_sort = System.nanoTime();
+ decodeTime_sort += ((e_sort - s_sort) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded_sort.length; i++) {
+ assertEquals(sortedData2[i], data2_arr_decoded_sort[i]);
+ }
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(data1.size()),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio),
+ String.valueOf(encodeTime_sort),
+ String.valueOf(decodeTime_sort),
+ String.valueOf(compressed_size_sort),
+ String.valueOf(ratio_sort),
+ };
+ writer.writeRecord(record);
+// System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOBlockSizeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOBlockSizeTest.java
new file mode 100644
index 0000000..92549ac
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOBlockSizeTest.java
@@ -0,0 +1,214 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongPSOBlockSizeTest {
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir =
+ // "D:/encoding-subcolumn/result/compression_vs_block_pso/";
+ String output_parent_dir = parent_dir + "result/compression_vs_block_pso/";
+
+ File outputDir = new File(output_parent_dir);
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ }
+
+ int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ int repeatTime = 100;
+ repeatTime = 600;
+
+ // repeatTime = 1;
+
+ // 定义数据集名称列表
+ List<String> datasetList = new ArrayList<>();
+ datasetList.add("Arade4");
+ datasetList.add("Bird-migration");
+ datasetList.add("Bitcoin-price");
+ datasetList.add("City-temp");
+ datasetList.add("Dewpoint-temp");
+ datasetList.add("EPM-Education");
+ datasetList.add("Gov10");
+ // datasetList.add("POI-lat");
+ datasetList.add("IR-bio-temp");
+ datasetList.add("PM10-dust");
+ datasetList.add("Stocks-DE");
+ datasetList.add("Stocks-UK");
+ datasetList.add("Stocks-USA");
+ datasetList.add("Wind-Speed");
+ datasetList.add("Wine-Tasting");
+
+ for (int block_size : block_size_list) {
+
+ String outputPath = output_parent_dir + "subcolumn_pso_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ // 使用数据集名称列表循环
+ for (String datasetName : datasetList) {
+ String filePath = input_parent_dir + datasetName + ".csv";
+ File file = new File(filePath);
+
+ // 检查文件是否存在
+ if (!file.exists()) {
+ System.out.println("File not found: " + filePath);
+ continue;
+ }
+
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongPSOTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = SubcolumnLongPSOTest.Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println("Block size: " + block_size);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOTest.java
new file mode 100644
index 0000000..efe8c37
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPSOTest.java
@@ -0,0 +1,1125 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongPSOTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+// ------------------ PSO-based Subcolumn (replace original Subcolumn) ------------------
+
+ /**
+ * Compute the total storage cost for a given beta following original cost logic.
+ * This mirrors the cost calculation in the original Subcolumn implementation.
+ */
+ private static int computeCostForBeta(long[] x, int x_length, int m, int block_size, int beta) {
+ // clamp beta to [1, m]
+ if (beta < 1) beta = 1;
+ if (beta > m) beta = m;
+
+ int l = (m + beta - 1) / beta;
+ long[][] subcolumnList = new long[l][x_length];
+ int[] bitWidthList = new int[l];
+
+ int maskBeta;
+ long mask;
+ // build subcolumns and bitWidthList
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta;
+ // compute mask safely (avoid shifting by >=64)
+ if (beta >= 63) {
+ mask = ~0L;
+ } else {
+ mask = ((1L << beta) - 1L);
+ }
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ int bw = bitWidth(block_size);
+ int totalCost = 0;
+
+ // for each sub-column compute min(bpCost, rleCost) following original logic
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthList[i] * x_length;
+ int rleCost = 0;
+
+ long currentNumber = subcolumnList[i][0];
+ int index = 0;
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+ // if intermediate RLE cost already >= bpCost, stop (same break condition)
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ totalCost += bpCost;
+ continue;
+ }
+
+ // finish computing run count (index currently = number of transitions)
+ index++;
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ totalCost += bpCost;
+ } else {
+ totalCost += rleCost;
+ }
+ }
+
+ return totalCost;
+ }
+
+ /**
+ * PSO main routine: search integer beta in [1, m] minimizing computeCostForBeta.
+ * Returns best integer beta found.
+ * <p>
+ * Parameters (tunable):
+ * - swarmSize: number of particles
+ * - maxIter: number of PSO iterations
+ * - w, c1, c2: PSO coefficients
+ * - localSearchRadius: after PSO, perform small local search within +/- radius
+ */
+ private static int psoFindBestBeta(long[] x, int x_length, int m, int block_size,
+ int swarmSize, int maxIter, double w, double c1, double c2, int localSearchRadius, long seed) {
+
+ if (m <= 1) {
+ return 1;
+ }
+
+ Random rand = (seed == 0) ? new Random() : new Random(seed);
+
+ // search range 1..m
+ double minPos = 1.0;
+ double maxPos = (double) m;
+
+ // particle arrays
+ double[] pos = new double[swarmSize];
+ double[] vel = new double[swarmSize];
+ double[] pbestPos = new double[swarmSize];
+ int[] pbestCost = new int[swarmSize];
+
+ // initialize particles
+ for (int i = 0; i < swarmSize; i++) {
+ pos[i] = minPos + rand.nextDouble() * (maxPos - minPos);
+ // initial velocity small random
+ vel[i] = (rand.nextDouble() - 0.5) * (maxPos - minPos) * 0.2;
+ int intval = (int) Math.round(pos[i]);
+ if (intval < 1) intval = 1;
+ if (intval > m) intval = m;
+ pbestPos[i] = pos[i];
+ pbestCost[i] = computeCostForBeta(x, x_length, m, block_size, intval);
+ }
+
+ // global best
+ int gbestIndex = 0;
+ int gbestCost = pbestCost[0];
+ double gbestPos = pbestPos[0];
+ for (int i = 1; i < swarmSize; i++) {
+ if (pbestCost[i] < gbestCost) {
+ gbestCost = pbestCost[i];
+ gbestPos = pbestPos[i];
+ gbestIndex = i;
+ }
+ }
+
+ double vmax = maxPos; // velocity clamp
+
+ // PSO iterations
+ for (int iter = 0; iter < maxIter; iter++) {
+ for (int i = 0; i < swarmSize; i++) {
+ double r1 = rand.nextDouble();
+ double r2 = rand.nextDouble();
+
+ // velocity update
+ vel[i] = w * vel[i]
+ + c1 * r1 * (pbestPos[i] - pos[i])
+ + c2 * r2 * (gbestPos - pos[i]);
+
+ // clamp velocity
+ if (vel[i] > vmax) vel[i] = vmax;
+ if (vel[i] < -vmax) vel[i] = -vmax;
+
+ // position update
+ pos[i] += vel[i];
+
+ // clamp position
+ if (pos[i] < minPos) {
+ pos[i] = minPos;
+ vel[i] = 0.0;
+ }
+ if (pos[i] > maxPos) {
+ pos[i] = maxPos;
+ vel[i] = 0.0;
+ }
+
+ // evaluate integer beta = round(pos)
+ int intval = (int) Math.round(pos[i]);
+ if (intval < 1) intval = 1;
+ if (intval > m) intval = m;
+
+ int cost = computeCostForBeta(x, x_length, m, block_size, intval);
+
+ // update pbest
+ if (cost < pbestCost[i]) {
+ pbestCost[i] = cost;
+ pbestPos[i] = pos[i];
+ // update gbest
+ if (cost < gbestCost) {
+ gbestCost = cost;
+ gbestPos = pos[i];
+ gbestIndex = i;
+ }
+ }
+ }
+ // optionally: you could add inertia damping or early stopping here if desired
+ }
+
+ // final integer best
+ int bestBeta = (int) Math.round(gbestPos);
+ if (bestBeta < 1) bestBeta = 1;
+ if (bestBeta > m) bestBeta = m;
+
+ // small local search around bestBeta to refine (try +/- localSearchRadius)
+ int bestCost = computeCostForBeta(x, x_length, m, block_size, bestBeta);
+ int start = Math.max(1, bestBeta - localSearchRadius);
+ int end = Math.min(m, bestBeta + localSearchRadius);
+ for (int b = start; b <= end; b++) {
+ int c = computeCostForBeta(x, x_length, m, block_size, b);
+ if (c < bestCost) {
+ bestCost = c;
+ bestBeta = b;
+ }
+ }
+
+ return bestBeta;
+ }
+
+ /**
+ * PSO-based Subcolumn entry point (replaces original Subcolumn).
+ * Uses default PSO hyperparameters similar to typical settings.
+ */
+ public static int Subcolumn(long[] x, int x_length, int m, int block_size) {
+ // PSO hyperparameters (you can tune these if needed)
+ final int SWARM_SIZE = 3;
+ final int MAX_ITER = 2;
+ final double W = 0.72; // inertia
+ final double C1 = 1.5; // cognitive
+ final double C2 = 1.5; // social
+ final int LOCAL_RADIUS = 3; // local search radius
+ final long SEED = 0L; // 0 -> use random seed; non-zero -> reproducible
+
+
+ // trivial cases
+ if (x_length == 0) return 1;
+ if (m <= 1) return 1;
+ m=4;
+ // run PSO to find best beta in [1, m]
+ int betaBest = psoFindBestBeta(x, x_length, m, block_size,
+ SWARM_SIZE, MAX_ITER, W, C1, C2, LOCAL_RADIUS, SEED);
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ long previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ long[] rle_values = new long[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, long[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("beta: " + beta[0]);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_pso.csv";
+
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns-PSO",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryBPTest.java
new file mode 100644
index 0000000..9c46d8a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryBPTest.java
@@ -0,0 +1,1177 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+public class SubcolumnLongPointQueryBPTest {
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnBP(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ long[][] subcolumnList = new long[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ // int rleCost = 0;
+
+ // // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+
+ // int index = 0;
+
+ // boolean bpBest = false;
+
+ // for (int j = 1; j < x_length; j++) {
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ // }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // index++;
+
+ // System.out.println("index: " + index);
+
+ // rleCost = bw * index + bitWidthListList[i] * index;
+
+ // // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += bpCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // int bpCost = bitWidthList[i] * list_length;
+ // int rleCost = 0;
+
+ // int previous = subcolumnList[i][0];
+ // int index = 0;
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // index++;
+ // previous = currentNumber;
+ // }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ // }
+
+ // index++;
+
+ // rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ // encodingType[i] = 1;
+
+ // encoded_result[encode_pos] = (byte) (index >> 8);
+ // encode_pos += 1;
+ // encoded_result[encode_pos] = (byte) (index & 0xFF);
+ // encode_pos += 1;
+
+ // index = 0;
+ // int[] run_length = new int[list_length];
+ // int[] rle_values = new int[list_length];
+ // previous = subcolumnList[i][0];
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // run_length[index] = j;
+ // rle_values[index] = previous;
+ // index++;
+ // previous = currentNumber;
+ // }
+ // }
+
+ // run_length[index] = list_length;
+ // rle_values[index] = previous;
+ // index++;
+
+ // encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ // encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ // if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ // } else {
+ // int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ // encode_pos += 2;
+
+ // int[] run_length = new int[index];
+ // int[] rle_values = new int[index];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ // int currentIndex = 0;
+ // for (int j = 0; j < index; j++) {
+ // int endPos = run_length[j];
+ // int value = rle_values[j];
+ // while (currentIndex < endPos) {
+ // subcolumnList[i][currentIndex] = value;
+ // currentIndex++;
+ // }
+ // }
+ // }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+// System.out.println(min_delta[0]);
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+// int beta = 2;
+// if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+// System.out.println(m);
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder, m, block_size);
+// }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ int pre_encode_pos = encode_pos;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2intBytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ long[] result = new long[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+// System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, long[] result) {
+
+ encode_pos += 2;
+
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+// System.out.println(min_delta[0]);
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+// System.out.println("m: "+m);
+// System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ long result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+// int type = encodingType[i];
+// if (type == 0) {
+ encode_pos *= 8;
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+// } else {
+//
+// int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+//
+// encode_pos += 2;
+//
+//
+// int[] run_length = new int[index];
+// int[] rle_values = new int[index];
+//
+// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+// encode_pos *= 8;
+// int accumulate_run_length = 0;
+// if(pos_in_cur_block == accumulate_run_length){
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos, bitWidthList[i]);
+// }else{
+// for(int x=0;x<index;x++){
+// accumulate_run_length += run_length[x];
+// if(pos_in_cur_block < accumulate_run_length ){
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos + x * bitWidthList[i], bitWidthList[i]);
+// break;
+// }
+// }
+// }
+// encode_pos += index * bitWidthList[i];
+// encode_pos = (encode_pos + 7) / 8;
+//
+//// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+//// rle_values);
+//
+//
+// }
+ result_value <<= beta;
+ }
+ result[0]= result_value+ min_delta[0];
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+// // String output_parent_dir = parent_dir + "result/";
+//
+// // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "subcolumn_bp.csv";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query_bp/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+ int[] beta_list = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31};
+ // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 100000;
+
+ // repeatTime = 1;
+// for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+// if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")||datasetName.equals("Gov10")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+// if(datasetName.equals("Bitcoin-price")){
+ data2_arr = new long[100000];
+ for(int j=0;j<100000;j++){
+ int i = j% data1.size();
+// for (int i = 0; i < data1.size(); i++) {
+ data2_arr[j] = (int) (data1.get(i) * max_mul);
+// }
+ }
+// }
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+// }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data2_arr.length * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ Random random = new Random();
+
+ // 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data2_arr.length /block_size*block_size;
+ long total_points = 0;
+ int randomNumber = max_random_value - (block_size/2); // block_size/2;//
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int randomNumber = random.nextInt(max_random_value);
+// total_points += (block_size - (max_random_value - randomNumber)%block_size);
+ Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(total_points/repeatTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+// }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryRLETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryRLETest.java
new file mode 100644
index 0000000..e48f71f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryRLETest.java
@@ -0,0 +1,1178 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+public class SubcolumnLongPointQueryRLETest {
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnRLE(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ long[][] subcolumnList = new long[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ long currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += rleCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ // int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ long previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ // encodingType[i] = 0;
+
+ // encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ long[] rle_values = new long[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result,int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+// int beta = 2;
+// if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder, m, block_size);
+// }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta[0], block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ int pre_encode_pos = encode_pos;
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result,beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2intBytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result,beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ long[] result = new long[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+// System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, long[] result) {
+
+ encode_pos += 2;
+
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+// System.out.println("m: "+m);
+// System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ long result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+// int type = encodingType[i];
+// if (type == 0) {
+// encode_pos *= 8;
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+//
+// encode_pos += remainder * bitWidthList[i];
+// encode_pos = (encode_pos + 7) / 8;
+
+// } else {
+//
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+
+ int[] run_length = new int[index];
+ long[] rle_values = new long[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos *= 8;
+ int accumulate_run_length = 0;
+ if(pos_in_cur_block == accumulate_run_length){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos, bitWidthList[i]);
+ }else{
+ for(int x=0;x<index;x++){
+ accumulate_run_length += run_length[x];
+ if(pos_in_cur_block < accumulate_run_length ){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + x * bitWidthList[i], bitWidthList[i]);
+ break;
+ }
+ }
+ }
+ encode_pos += index * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+//// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+//// rle_values);
+//
+//
+// }
+ result_value <<= beta;
+ }
+ result[0]= result_value + min_delta[0];
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+// // String output_parent_dir = parent_dir + "result/";
+//
+// // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "subcolumn_rle.csv";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query_rle/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+ int[] beta_list = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31}; // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 10000;
+
+ // repeatTime = 1;
+// for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+// if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")||datasetName.equals("Gov10")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+// if(datasetName.equals("Bitcoin-price")){
+// data2_arr = new long[data1.size()*13];
+// for(int j=0;j<13;j++){
+// for (int i = 0; i < data1.size(); i++) {
+// data2_arr[i+j*data1.size()] = (int) (data1.get(i) * max_mul);
+// }
+// }
+// }
+ data2_arr = new long[100000];
+ for(int j=0;j<100000;j++){
+ int i = j% data1.size();
+// for (int i = 0; i < data1.size(); i++) {
+ data2_arr[j] = (int) (data1.get(i) * max_mul);
+// }
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 16];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+// }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data2_arr.length * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ Random random = new Random();
+ // 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data2_arr.length/block_size*block_size;
+ int randomNumber = max_random_value - (block_size/2); //random.nextInt(max_random_value);max_random_value - (block_size*3/2);
+
+ s = System.nanoTime();
+ long total_points = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int randomNumber = random.nextInt(max_random_value);
+// total_points += (block_size - (max_random_value - randomNumber)%block_size);
+ Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(total_points/repeatTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+// }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryTest.java
new file mode 100644
index 0000000..f3c7d12
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongPointQueryTest.java
@@ -0,0 +1,887 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class SubcolumnLongPointQueryTest {
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ long previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ long[] rle_values = new long[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result,int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+// int beta[0] = 2;
+// if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+// System.out.println(m);
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder, m, block_size);
+// }
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+// System.out.println(Arrays.toString(beta));
+
+ int pre_encode_pos = encode_pos;
+
+// encode_pos += 2;
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result,beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2intBytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result,beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ }
+
+ return encode_pos;
+ }
+
+
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ long[] result = new long[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+// System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, long[] result) {
+
+ encode_pos += 2;
+
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+// System.out.println("m: "+m);
+// System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ int result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ encode_pos *= 8;
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos *= 8;
+ int accumulate_run_length = 0;
+ if(pos_in_cur_block == accumulate_run_length){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos, bitWidthList[i]);
+ }else{
+ for(int x=0;x<index;x++){
+ accumulate_run_length += run_length[x];
+ if(pos_in_cur_block < accumulate_run_length ){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + x * bitWidthList[i], bitWidthList[i]);
+ break;
+ }
+ }
+ }
+ encode_pos += index * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+// rle_values);
+
+
+ }
+ result_value <<= beta;
+ }
+ result[0]= result_value + min_delta[0];
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ @Test
+ public void testQueryBeta() throws IOException {
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 10000;
+
+// repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+// if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+// if(datasetName.equals("Bitcoin-price")){
+// data2_arr = new long[data1.size()*13];
+// for(int j=0;j<13;j++){
+// for (int i = 0; i < data1.size(); i++) {
+// data2_arr[i+j*data1.size()] = (int) (data1.get(i) * max_mul);
+// }
+// }
+// }
+ data2_arr = new long[100000];
+ for(int j=0;j<100000;j++){
+ int i = j% data1.size();
+// for (int i = 0; i < data1.size(); i++) {
+ data2_arr[j] = (int) (data1.get(i) * max_mul);
+// }
+ }
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ length = Encoder(data2_arr, block_size, encoded_result);
+
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data2_arr.length * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data2_arr.length * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ Random random = new Random();
+
+ // 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data2_arr.length/block_size*block_size;
+ long total_points = 0;
+ int randomNumber =max_random_value - (block_size/2) ;// block_size/2;// ;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int randomNumber = random.nextInt(max_random_value);
+// total_points += (block_size - (max_random_value - randomNumber)%block_size);
+ Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(total_points/repeatTime),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryCountTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryCountTest.java
new file mode 100644
index 0000000..dce90b3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryCountTest.java
@@ -0,0 +1,224 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnLongQueryCountTest {
+
+ public static void Query(byte[] encoded_result, int target) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuery(encoded_result, i, block_size,
+ block_size, encode_pos, target,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ if (value == target) {
+ result[result_length[0]]++;
+ }
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, target,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int target, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ // min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ // encode_pos += 4;
+
+ min_delta[0] = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ target -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 target 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ result[result_length[0]] += remainder;
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ int current = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][index] < value) {
+ // result[result_length[0]] = block_size * block_index + index;
+ // result_length[0]++;
+ // } else if (subcolumnList[i][index] == value) {
+ // candidate_indices[new_length] = index;
+ // new_length++;
+ // }
+ if (current == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ // if (rle_values[rleIndex] < value) {
+ // result[result_length[0]] = block_size * block_index + index_candidate;
+ // result_length[0]++;
+ // } else if (rle_values[rleIndex] == value) {
+ // candidate_indices[new_length] = index_candidate;
+ // new_length++;
+ // }
+ if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ // if (target <= 0) {
+ // for (int i = 0; i < remainder; i++) {
+ // result[result_length[0]] = block_size * block_index + i;
+ // result_length[0]++;
+ // }
+ // return encode_pos;
+ // }
+
+ result[result_length[0]] += candidate_length;
+
+ return encode_pos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryEqualTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryEqualTest.java
new file mode 100644
index 0000000..acc8017
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQueryEqualTest.java
@@ -0,0 +1,226 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnLongQueryEqualTest {
+
+ public static void Query(byte[] encoded_result, int target) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuery(encoded_result, i, block_size,
+ block_size, encode_pos, target,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ long value = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ if (value == target) {
+ result[result_length[0]] = block_size * num_blocks + i;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, target,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int target, int[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ // min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ min_delta[0] = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ target -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 target 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][index] < value) {
+ // result[result_length[0]] = block_size * block_index + index;
+ // result_length[0]++;
+ // } else if (subcolumnList[i][index] == value) {
+ // candidate_indices[new_length] = index;
+ // new_length++;
+ // }
+ if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ // if (rle_values[rleIndex] < value) {
+ // result[result_length[0]] = block_size * block_index + index_candidate;
+ // result_length[0]++;
+ // } else if (rle_values[rleIndex] == value) {
+ // candidate_indices[new_length] = index_candidate;
+ // new_length++;
+ // }
+ if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ // if (target <= 0) {
+ // for (int i = 0; i < remainder; i++) {
+ // result[result_length[0]] = block_size * block_index + i;
+ // result_length[0]++;
+ // }
+ // return encode_pos;
+ // }
+ for (int i = 0; i < candidate_length; i++) {
+ result[result_length[0]] = block_size * block_index + candidate_indices[i];
+ result_length[0]++;
+ }
+
+ return encode_pos;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQuerySum2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQuerySum2Test.java
new file mode 100644
index 0000000..2967a9a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongQuerySum2Test.java
@@ -0,0 +1,201 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnLongQuerySum2Test {
+
+ public static void Query(byte[] encoded_result, long target) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ long[] result = new long[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuery(encoded_result, i, block_size, block_size, encode_pos, target, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 8;
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ } else {
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size, remainder, encode_pos, target,
+ result, result_length);
+ }
+
+ // for (int i = 0; i < result_length[0]; i++) {
+ // System.out.print(result[i] + " ");
+ // }
+ // System.out.println();
+
+ }
+
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long target, long[] result, int[] result_length) {
+ long[] min_delta = new long[3];
+
+ // min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ // encode_pos += 4;
+
+ min_delta[0] = SubcolumnLongTest.bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ target -= min_delta[0];
+
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ result[result_length[0]] = min_delta[0] * remainder;
+ result_length[0]++;
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnLongTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnLongTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (int) (target >> (i * beta)) & ((1 << beta) - 1);
+
+ if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ long value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+ }
+ }
+
+ result[result_length[0]] = (min_delta[0] + target) * candidate_length;
+ result_length[0]++;
+
+ return encode_pos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongRLETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongRLETest.java
new file mode 100644
index 0000000..799e624
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongRLETest.java
@@ -0,0 +1,996 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnLongRLETest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Long
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
+
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnRLE(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ long[][] subcolumnList = new long[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ long currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += rleCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ long maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ // encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ // int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ long previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ // encodingType[i] = 0;
+
+ // encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ long[] rle_values = new long[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ long currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+ }
+
+ // preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, long[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ long[][] subcolumnList = new long[l][list_length];
+
+ // int[] encodingType = new int[l];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ // int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ // if (type == 0) {
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ // subcolumnList[i]);
+ // } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ long[] rle_values = new long[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ long value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ // }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ // beta[0] = SubcolumnRLE(data_delta, remainder, m, block_size);
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("beta: " + beta[0]);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_long_rle_repeat100.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ // repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 59];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongTest.java
new file mode 100644
index 0000000..7f8f36a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnLongTest.java
@@ -0,0 +1,954 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnLongTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void longToBytes(long srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1L << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static long bytesToLong(byte[] result, int pos, int width) {
+ long ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void pack8Values(
+ long[] values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8Values(
+ byte[] encoded, int offset, int width, long[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = buffer >>> (totalBits - width);
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int bitPacking(long[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ longToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, long[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToLong(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(long[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
+ // int[] beta_list = { 2, 3, 4, 7, 11 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta = 1; beta < 5; beta++) {
+ // for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (int) ((x[j] >> (i * beta)) & ((1 << beta) - 1));
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(long[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ long maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (int) ((list[j] >> shiftAmount) & mask);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, long[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= (long) subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ long cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("beta: " + beta[0]);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] block_data = new long[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/"; //
+
+ String outputPath = output_parent_dir + "subcolumn_long0.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ for (int i = 0; i < 10; i++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterialize.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterialize.java
new file mode 100644
index 0000000..429dc64
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterialize.java
@@ -0,0 +1,1346 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Arrays;
+
+
+public class SubcolumnMaterialize {
+
+ public static void QueryTwoColumns(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2) {
+ int[] first_column_results = new int[encoded_result1.length];
+ int[] first_result_length = new int[1];
+
+ Query(encoded_result1, upper_bound1, first_column_results, first_result_length);
+
+ int[] final_results = new int[first_result_length[0]];
+ int[] final_result_length = new int[1];
+
+ QueryWithIndices(encoded_result2, upper_bound2, first_column_results, first_result_length[0],
+ final_results, final_result_length);
+ }
+
+ public static void QueryWithIndices(byte[] encoded_result, int upper_bound,
+ int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ | ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 初始化结果索引
+ result_length[0] = 0;
+
+ int[] blockIndicesCount = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ blockIndicesCount[blockIndex]++;
+ }
+
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int i = 0; i <= num_blocks; i++) {
+ blockIndices[i] = new int[blockIndicesCount[i]];
+ }
+
+ int[] currentIndices = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ int localIndex = index % block_size;
+
+ blockIndices[blockIndex][currentIndices[blockIndex]] = localIndex;
+ currentIndices[blockIndex]++;
+ }
+
+ // 遍历所有块
+ for (int i = 0; i < num_blocks; i++) {
+
+ if (blockIndicesCount[i] == 0) {
+ // 计算跳过此块所需的字节数
+ encode_pos = SkipBlock(encoded_result, i, block_size,
+ block_size, encode_pos);
+ continue;
+ }
+
+ // 对该块中的候选索引执行查询
+ encode_pos = BlockQueryWithIndices(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ blockIndices[i], blockIndicesCount[i], result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder > 0) {
+ if (blockIndicesCount[num_blocks] > 0) {
+ if (remainder <= 3) {
+ for (int j = 0; j < blockIndicesCount[num_blocks]; j++) {
+ int idx = blockIndices[num_blocks][j];
+ int offset = num_blocks * block_size + idx;
+ if (offset < data_length) {
+ int value = ((encoded_result[encode_pos + idx * 4] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + idx * 4 + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + idx * 4 + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + idx * 4 + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = offset;
+ result_length[0]++;
+ }
+ }
+ }
+ encode_pos += remainder * 4;
+ } else {
+
+ encode_pos = BlockQueryWithIndices(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ blockIndices[num_blocks], blockIndicesCount[num_blocks], result, result_length);
+ }
+ } else {
+ // 没有候选索引，跳过剩余部分
+ if (remainder <= 3) {
+ encode_pos += remainder * 4;
+ } else {
+ encode_pos = SkipBlock(encoded_result, num_blocks, block_size,
+ remainder, encode_pos);
+ }
+ }
+ }
+ }
+
+ public static int BlockQueryWithIndices(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 所有索引默认都是候选索引
+ int[] filtered_indices = new int[candidate_length];
+ int filtered_length = candidate_length;
+ System.arraycopy(candidate_indices, 0, filtered_indices, 0, candidate_length);
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < filtered_length; i++) {
+ result[result_length[0]] = block_size * block_index + filtered_indices[i];
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < filtered_length; j++) {
+ int index = filtered_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ filtered_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ filtered_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ // 为每个候选索引查找对应的RLE值
+ for (int j = 0; j < filtered_length; j++) {
+ int index_candidate = filtered_indices[j];
+
+ // 查找包含此索引的RLE段
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ filtered_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ filtered_length = new_length;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ private static int SkipBlock(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // int[] min_delta = new int[3];
+
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8;
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ encode_pos = (encode_pos * 8 + bw * index + 7) / 8;
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * index + 7) / 8;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static void Query(byte[] encoded_result, int upper_bound, int[] result, int[] result_length) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ // int[] result = new int[data_length];
+ // int[] result_length = new int[1];
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 upper_bound 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
+ // String output_parent_dir = parent_dir + "result/query_vs_block/";
+
+ int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ for (int block_size : block_size_list) {
+ String outputPath = output_parent_dir + "subcolumn_query_less_parts_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ // 创建两个数据列
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+
+ // 填充第一列
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // 填充第二列
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 4];
+ byte[] encoded_result2 = new byte[col2_data.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ // 编码第一列
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = SubcolumnTest.Encoder(col1_data, block_size, encoded_result1);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 编码第二列
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length2 = SubcolumnTest.Encoder(col2_data, block_size, encoded_result2);
+ }
+ e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size = length1 + length2;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ QueryTwoColumns(encoded_result1, encoded_result2,
+ queryRange.get(datasetName), queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("block_size: " + block_size);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ @Test
+ public void testQueryBeta() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int block_size = 512;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_query_less_parts_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ // 创建两个数据列
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+
+ // 填充第一列
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // 填充第二列
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 4];
+ byte[] encoded_result2 = new byte[col2_data.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ // 编码第一列
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = SubcolumnBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 编码第二列
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length2 = SubcolumnBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ }
+ e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size = length1 + length2;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ QueryTwoColumns(encoded_result1, encoded_result2,
+ queryRange.get(datasetName), queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+ // 放在类的末尾，作为一个新的测试 / helper
+ @Test
+ public void compareMaterializationStrategies() throws IOException {
+ // --- 基本设置，复用你 testQuery 中的路径 / 数据准备逻辑 ---
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/materialization/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+
+// // 这里为了演示，仅处理单个 CSV 文件（你可以循环多个文件）
+// File directory = new File(input_parent_dir);
+// File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+// if (csvFiles == null || csvFiles.length == 0) {
+// System.out.println("No csv files found under " + input_parent_dir);
+// return;
+// }
+//
+// // 选第一个文件作为 demo
+// File file = csvFiles[0];
+// String datasetName = extractFileName(file.toString());
+// System.out.println("Dataset: " + datasetName);
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 500;
+ String outputPath = output_parent_dir + "subcolumn_materialization.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "LM-pipelined",
+ "LM-parallel",
+ "EM-pipelined",
+ "EM-parallel",
+// "Encoding Time",
+// "Decoding Time",
+ "Points",
+// "Compressed Size",
+// "Compression Ratio"
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // 读取列并构造两列（复用你已有的读取逻辑）
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(!queryRange.containsKey(datasetName))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> raw = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ raw.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = raw.size();
+ int halfSize = totalSize / 2;
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) col1_data[i] = (int) (raw.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++) col2_data[i] = (int) (raw.get(i + halfSize) * max_mul);
+
+ int block_size = 512; // 选择一个 block size 做比较
+// int repeatTime = 200;
+ byte[] encoded_result1 = new byte[col1_data.length * 4];
+ byte[] encoded_result2 = new byte[col2_data.length * 4];
+
+ // 编码（复用你的 Encoder）
+ int length1 = SubcolumnTest.Encoder(col1_data, block_size, encoded_result1);
+ int length2 = SubcolumnTest.Encoder(col2_data, block_size, encoded_result2);
+
+ int upper = queryRange.containsKey(datasetName) ? queryRange.get(datasetName) : Integer.MAX_VALUE;
+
+// System.out.println("Running repeats: " + repeatTime + " upper=" + upper);
+
+ // ---------- 1) LM-pipelined: your existing QueryTwoColumns ----------
+ long tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ QueryTwoColumns(encoded_result1, encoded_result2, upper, upper);
+ }
+ long tEnd = System.nanoTime();
+ long lmPipelinedTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-pipelined avg ns: " + lmPipelinedTime);
+
+ // ---------- 2) LM-parallel: Query both columns separately -> intersect positions ----------
+ // helper arrays reused
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ // warm run to avoid JIT one-time overhead bias
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ // intersect result sets (they are arrays of positions)
+ // 我们用 BitSet 做交集，对长度总点数即 halfSize 做 size
+ BitSet b1 = new BitSet(halfSize);
+ BitSet b2 = new BitSet(halfSize);
+ for (int i = 0; i < len1[0]; i++) b1.set(res1[i]);
+ for (int i = 0; i < len2[0]; i++) b2.set(res2[i]);
+ b1.and(b2); // intersection
+ int match = b1.cardinality(); // 匹配数量
+ // 如果你想收集 matched positions，可以用 b1.stream() 或 nextSetBit
+ }
+ tEnd = System.nanoTime();
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+ int[] result = new int[encoded_result1.length];
+ int[] resultLen = new int[1];
+ // ---------- 3) EM-pipelined (近似实现说明) ----------
+ // 说明：严格的 EM-pipelined 需要把第一列物化成 (pos,value) tuples（即完整解码得到值），
+ // 然后按这些 pos 去第二列延展并筛选。要做到严格，需实现「按索引只解码第二列对应位置的值」或把第二列解码成数组。
+ // 在这里给出一个“可运行的近似实现”：把第一列先用 Query() 拿到候选 positions（pos），
+ // 再把这些 pos 作为 candidate 传入 QueryWithIndices(encoded_result2,...)
+ // （注意：这是 LM-pipelined 与 EM-pipelined 在含义上并非完全相同，但在当前可用接口下这是能运行且可比较的实现）
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // 获取第一列候选位置（认为已“物化”成pos list）
+ resultLen[0] = 0;
+ QueryTwoColumnsStrictEMPipelined(encoded_result1, encoded_result2, queryRange.get(datasetName), queryRange.get(datasetName), result, resultLen);
+
+// Query(encoded_result1, upper, res1, len1);
+// // 用这些位置去查询第二列（QueryWithIndices 将仅在这些位置上做判断）
+// QueryWithIndices(encoded_result2, upper, res1, len1[0], res2, len2);
+// // res2 中现在是满足第二列 < upper 的偏移（相对于全列偏移），如果需要对第一列的值也做判定，需要把第一列解码成values，这里省略
+ }
+ tEnd = System.nanoTime();
+ long emPipelinedApproxTime = (tEnd - tStart) / repeatTime;
+// System.out.println("EM-pipelined strict matched: " + resultLen[0]);
+ System.out.println("EM-pipelined (approx) avg ns: " + emPipelinedApproxTime);
+
+ // ---------- 4) EM-parallel ----------
+
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // 近似实现：同时对两列调用 QueryWithIndices（先对第一列获取候选pos，然后把这些pos作为输入去第二列）
+ QueryTwoColumnsStrictEMParallel(encoded_result1, encoded_result2, queryRange.get(datasetName), queryRange.get(datasetName), result, resultLen);
+// Query(encoded_result1, upper, res1, len1); // first column candidates
+// QueryWithIndices(encoded_result2, upper, res1, len1[0], res2, len2);
+ // 结果 res2 表示在那些 pos 中满足第二列条件的偏移
+ }
+ tEnd = System.nanoTime();
+// System.out.println("EM-parallel strict matched: " + resultLen[0]);
+ long emParallelApproxTime = (tEnd - tStart) / repeatTime;
+ System.out.println("EM-parallel (approx) avg ns: " + emParallelApproxTime);
+
+ // 最后打印一行小结
+ System.out.println("Summary (ns avg per query): LM-pipelined=" + lmPipelinedTime
+ + " LM-parallel=" + lmParallelTime
+ + " EM-pipelined-approx=" + emPipelinedApproxTime
+ + " EM-parallel-approx=" + emParallelApproxTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(lmPipelinedTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(emPipelinedApproxTime),
+ String.valueOf(emParallelApproxTime),
+ String.valueOf(totalSize)
+ };
+ writer.writeRecord(record);
+
+ }
+ writer.close();
+ }
+
+ static class DecodedBlock {
+ int[] values;
+ int newEncodePos;
+ DecodedBlock(int[] values, int newEncodePos) { this.values = values; this.newEncodePos = newEncodePos; }
+ }
+
+ /**
+ * 读 big-endian 32-bit int（与 Query/BlockQueryIndex 中的读取一致）。
+ */
+ private static int readInt32BE(byte[] arr, int pos) {
+ return ((arr[pos] & 0xFF) << 24) | ((arr[pos + 1] & 0xFF) << 16) | ((arr[pos + 2] & 0xFF) << 8)
+ | (arr[pos + 3] & 0xFF);
+ }
+
+ /**
+ * 严格地完整解码一个块（反向实现 BlockQueryIndex/WithIndices 的解析逻辑）。
+ *
+ * @param encoded_result 整列的字节数组
+ * @param block_index 块索引（仅用于可读性/日志，函数内部不用它定位）
+ * @param block_size 标称块大小（用于计算 bw）
+ * @param remainder 本块的元素数（最后一个块可能小于 block_size）
+ * @param encode_pos 当前字节偏移（函数会从这里读取，并返回更新后的字节偏移）
+ * @return DecodedBlock，包含本块每个位置的解码整型值（长度 = remainder）以及新的 encode_pos
+ */
+ public static DecodedBlock decodeBlockValues(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // 1) 读取 min_delta
+ int min_delta = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ // 2) 读取 m
+ int m = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 如果 m == 0：表示无 subcolumns（所有值等于 min_delta）
+ if (m == 0) {
+ int[] vals = new int[remainder];
+ if (remainder > 0) Arrays.fill(vals, min_delta);
+ return new DecodedBlock(vals, encode_pos);
+ }
+
+ // 3) 其他元信息
+ int bw = SubcolumnTest.bitWidth(block_size); // 基本宽度，用于 RLE run-length 的位宽
+ int beta = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ // 4) 读取 bitWidthList（每个 subcolumn 的位宽，使用 decodeBitPacking(bits=8)）
+ int[] bitWidthList = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // 5) 读取 encodingType（每个 subcolumn 的编码类型：0=bitpacked, 1=RLE）
+ int[] encodingType = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ // 6) 逐个 subcolumn（从高位 i=l-1 到 i=0）合成值
+ int[] blockValues = new int[remainder];
+ Arrays.fill(blockValues, 0);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bw_i = bitWidthList[i];
+
+ if (type == 0) {
+ // bitpacked: 在当前 encode_pos (字节偏移) 的位流上连续存 remainder 个 bw_i 位值
+ int bitStart = encode_pos * 8; // 转为位偏移
+ // 对每个位置抽取 bw_i 位并左移累加
+ for (int p = 0; p < remainder; p++) {
+ int bitOffset = bitStart + p * bw_i;
+ int part = SubcolumnTest.bytesToInt(encoded_result, bitOffset, bw_i);
+ blockValues[p] |= (part << (i * beta));
+ }
+ // 跳过这段 bitpacked 的位段，回到下一个字节边界
+ encode_pos = (bitStart + remainder * bw_i + 7) / 8;
+ } else {
+ // RLE: 先读 runCount (2 bytes)，随后是 run_length[]（bw 位）和 rle_values[]（bw_i 位）
+ int runCount = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ // 读取 run_length（每项 bw 位）
+ int[] run_length = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, runCount, run_length);
+
+ // 读取 rle_values（每项 bw_i 位）
+ int[] rle_values = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw_i, runCount, rle_values);
+
+ // 根据 run_lengths 展开并赋值
+ int pos = 0;
+ for (int r = 0; r < runCount && pos < remainder; r++) {
+ int len = run_length[r];
+ int val = rle_values[r];
+ for (int t = 0; t < len && pos < remainder; t++, pos++) {
+ blockValues[pos] |= (val << (i * beta));
+ }
+ }
+ // 注意：encode_pos 已由 decodeBitPacking 更新
+ }
+ }
+
+ // 7) 把 min_delta 加回每个位置
+ for (int p = 0; p < remainder; p++) {
+ blockValues[p] += min_delta;
+ }
+
+ return new DecodedBlock(blockValues, encode_pos);
+ }
+
+ /**
+ * 完整解码整列（逐块调用 decodeBlockValues）
+ *
+ * @param encoded_result 编码后字节数组（包含前 8 字节 header: data_length, block_size）
+ * @return 解码后的整列 int[]，长度 = data_length
+ */
+ public static int[] decodeColumnFully(byte[] encoded_result) {
+ int encode_pos = 0;
+ int data_length = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+ int block_size = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+ int[] out = new int[data_length];
+
+ // 完整块
+ for (int b = 0; b < num_blocks; b++) {
+ DecodedBlock db = decodeBlockValues(encoded_result, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, b * block_size, block_size);
+ }
+
+ // 最后一块（如果有剩余）
+ if (remainder > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result, num_blocks, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, num_blocks * block_size, remainder);
+ }
+
+ return out;
+ }
+
+ /**
+ * 严格 EM-parallel: 彻底解码两列为 int[]，然后逐行判断两个谓词同时成立的位置。
+ *
+ * @param encoded_result1 列1的编码字节数组
+ * @param encoded_result2 列2的编码字节数组
+ * @param upper_bound1 列1的上界谓词（< upper_bound1）
+ * @param upper_bound2 列2的上界谓词（< upper_bound2）
+ * @param result 用于输出匹配位置的数组（全表偏移 / 行号）
+ * @param result_length 长度容器（长度为1的数组，写回匹配数）
+ */
+ public static void QueryTwoColumnsStrictEMParallel(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 完整解码两列
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int[] col2 = decodeColumnFully(encoded_result2);
+
+ // 确定行数（取两列最小）
+ int n = Math.min(col1.length, col2.length);
+ result_length[0] = 0;
+
+ for (int i = 0; i < n; i++) {
+ if (col1[i] < upper_bound1 && col2[i] < upper_bound2) {
+ result[result_length[0]] = i;
+ result_length[0]++;
+ }
+ }
+ }
+
+ /**
+ * 严格 EM-pipelined: 完整解码第一列（物化为 values），根据第一列筛出 candidate positions（按块组织），
+ * 然后对第二列按块按需解码（只解包含 candidate 的块），在这些块中对 candidate 的局部索引做精确判定。
+ *
+ * @param encoded_result1 列1编码
+ * @param encoded_result2 列2编码
+ * @param upper_bound1
+ * @param upper_bound2
+ * @param result
+ * @param result_length
+ */
+ public static void QueryTwoColumnsStrictEMPipelined(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 1) 解码第 1 列（完整解码以物化 tuple 的该属性）
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int data_length = col1.length;
+
+ // 2) 构建候选位置分块索引（与 QueryWithIndices 中的策略一致）
+ int block_size = readInt32BE(encoded_result2, 4); // 注意：编码头部：前4字节 data_length，接着 4 字节 block_size
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 统计每个块中 candidate 数量
+ int[] blockCount = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ blockCount[bidx]++;
+ }
+ }
+
+ // 若没有 candidate，快速返回
+ int totalCandidates = 0;
+ for (int c : blockCount) totalCandidates += c;
+ result_length[0] = 0;
+ if (totalCandidates == 0) return;
+
+ // 为每块分配数组以记录 local indices
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int b = 0; b <= num_blocks; b++) {
+ blockIndices[b] = new int[blockCount[b]];
+ }
+ int[] cursor = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ int local = i % block_size;
+ blockIndices[bidx][cursor[bidx]++] = local;
+ }
+ }
+
+ // 3) 逐块扫描第二列：若该块没有 candidate -> 跳过（SkipBlock）；否则解码该块并测试
+ int encode_pos = 0;
+ int data_len_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ int bs_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ // sanity check bs_from_header == block_size
+ // 逐块循环
+ for (int b = 0; b < num_blocks; b++) {
+ if (blockCount[b] == 0) {
+ // 跳过此块
+ encode_pos = SkipBlock(encoded_result2, b, block_size, block_size, encode_pos);
+ continue;
+ }
+ // 需要解码此块
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ // 对该块的候选局部索引测试第二列谓词
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ }
+
+ // 最后可能的不完整块
+ if (remainder > 0) {
+ int b = num_blocks;
+ if (blockCount[b] > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ } else {
+ // 无 candidate，跳过或不处理
+ }
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterializeR1D8.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterializeR1D8.java
new file mode 100644
index 0000000..fd54902
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaterializeR1D8.java
@@ -0,0 +1,1038 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+public class SubcolumnMaterializeR1D8 {
+
+ public static void QueryTwoColumns(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2) {
+ int[] first_column_results = new int[encoded_result1.length];
+ int[] first_result_length = new int[1];
+
+ Query(encoded_result1, upper_bound1, first_column_results, first_result_length);
+
+ int[] final_results = new int[first_result_length[0]];
+ int[] final_result_length = new int[1];
+
+ QueryWithIndices(encoded_result2, upper_bound2, first_column_results, first_result_length[0],
+ final_results, final_result_length);
+ }
+
+ public static void QueryWithIndices(byte[] encoded_result, int upper_bound,
+ int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ | ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 初始化结果索引
+ result_length[0] = 0;
+
+ int[] blockIndicesCount = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ blockIndicesCount[blockIndex]++;
+ }
+
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int i = 0; i <= num_blocks; i++) {
+ blockIndices[i] = new int[blockIndicesCount[i]];
+ }
+
+ int[] currentIndices = new int[num_blocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / block_size;
+ int localIndex = index % block_size;
+
+ blockIndices[blockIndex][currentIndices[blockIndex]] = localIndex;
+ currentIndices[blockIndex]++;
+ }
+
+ // 遍历所有块
+ for (int i = 0; i < num_blocks; i++) {
+
+ if (blockIndicesCount[i] == 0) {
+ // 计算跳过此块所需的字节数
+ encode_pos = SkipBlock(encoded_result, i, block_size,
+ block_size, encode_pos);
+ continue;
+ }
+
+ // 对该块中的候选索引执行查询
+ encode_pos = BlockQueryWithIndices(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ blockIndices[i], blockIndicesCount[i], result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder > 0) {
+ if (blockIndicesCount[num_blocks] > 0) {
+ if (remainder <= 3) {
+ for (int j = 0; j < blockIndicesCount[num_blocks]; j++) {
+ int idx = blockIndices[num_blocks][j];
+ int offset = num_blocks * block_size + idx;
+ if (offset < data_length) {
+ int value = ((encoded_result[encode_pos + idx * 4] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + idx * 4 + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + idx * 4 + 2] & 0xFF) << 8) |
+ (encoded_result[encode_pos + idx * 4 + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = offset;
+ result_length[0]++;
+ }
+ }
+ }
+ encode_pos += remainder * 4;
+ } else {
+
+ encode_pos = BlockQueryWithIndices(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ blockIndices[num_blocks], blockIndicesCount[num_blocks], result, result_length);
+ }
+ } else {
+ // 没有候选索引，跳过剩余部分
+ if (remainder <= 3) {
+ encode_pos += remainder * 4;
+ } else {
+ encode_pos = SkipBlock(encoded_result, num_blocks, block_size,
+ remainder, encode_pos);
+ }
+ }
+ }
+ }
+
+ public static int BlockQueryWithIndices(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 所有索引默认都是候选索引
+ int[] filtered_indices = new int[candidate_length];
+ int filtered_length = candidate_length;
+ System.arraycopy(candidate_indices, 0, filtered_indices, 0, candidate_length);
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < filtered_length; i++) {
+ result[result_length[0]] = block_size * block_index + filtered_indices[i];
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < filtered_length; j++) {
+ int index = filtered_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ filtered_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ filtered_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ // 为每个候选索引查找对应的RLE值
+ for (int j = 0; j < filtered_length; j++) {
+ int index_candidate = filtered_indices[j];
+
+ // 查找包含此索引的RLE段
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ filtered_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ filtered_length = new_length;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ private static int SkipBlock(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // int[] min_delta = new int[3];
+
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8;
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ encode_pos = (encode_pos * 8 + bw * index + 7) / 8;
+ encode_pos = (encode_pos * 8 + bitWidthList[i] * index + 7) / 8;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static void Query(byte[] encoded_result, int upper_bound, int[] result, int[] result_length) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ // int[] result = new int[data_length];
+ // int[] result_length = new int[1];
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encode_pos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 upper_bound 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+
+ static class DecodedBlock {
+ int[] values;
+ int newEncodePos;
+ DecodedBlock(int[] values, int newEncodePos) { this.values = values; this.newEncodePos = newEncodePos; }
+ }
+
+ /**
+ * 读 big-endian 32-bit int（与 Query/BlockQueryIndex 中的读取一致）。
+ */
+ private static int readInt32BE(byte[] arr, int pos) {
+ return ((arr[pos] & 0xFF) << 24) | ((arr[pos + 1] & 0xFF) << 16) | ((arr[pos + 2] & 0xFF) << 8)
+ | (arr[pos + 3] & 0xFF);
+ }
+
+ /**
+ * 严格地完整解码一个块（反向实现 BlockQueryIndex/WithIndices 的解析逻辑）。
+ *
+ * @param encoded_result 整列的字节数组
+ * @param block_index 块索引（仅用于可读性/日志，函数内部不用它定位）
+ * @param block_size 标称块大小（用于计算 bw）
+ * @param remainder 本块的元素数（最后一个块可能小于 block_size）
+ * @param encode_pos 当前字节偏移（函数会从这里读取，并返回更新后的字节偏移）
+ * @return DecodedBlock，包含本块每个位置的解码整型值（长度 = remainder）以及新的 encode_pos
+ */
+ public static DecodedBlock decodeBlockValues(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ // 1) 读取 min_delta
+ int min_delta = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ // 2) 读取 m
+ int m = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ // 如果 m == 0：表示无 subcolumns（所有值等于 min_delta）
+ if (m == 0) {
+ int[] vals = new int[remainder];
+ if (remainder > 0) Arrays.fill(vals, min_delta);
+ return new DecodedBlock(vals, encode_pos);
+ }
+
+ // 3) 其他元信息
+ int bw = SubcolumnTest.bitWidth(block_size); // 基本宽度，用于 RLE run-length 的位宽
+ int beta = encoded_result[encode_pos] & 0xFF;
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ // 4) 读取 bitWidthList（每个 subcolumn 的位宽，使用 decodeBitPacking(bits=8)）
+ int[] bitWidthList = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // 5) 读取 encodingType（每个 subcolumn 的编码类型：0=bitpacked, 1=RLE）
+ int[] encodingType = new int[l];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ // 6) 逐个 subcolumn（从高位 i=l-1 到 i=0）合成值
+ int[] blockValues = new int[remainder];
+ Arrays.fill(blockValues, 0);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bw_i = bitWidthList[i];
+
+ if (type == 0) {
+ // bitpacked: 在当前 encode_pos (字节偏移) 的位流上连续存 remainder 个 bw_i 位值
+ int bitStart = encode_pos * 8; // 转为位偏移
+ // 对每个位置抽取 bw_i 位并左移累加
+ for (int p = 0; p < remainder; p++) {
+ int bitOffset = bitStart + p * bw_i;
+ int part = SubcolumnTest.bytesToInt(encoded_result, bitOffset, bw_i);
+ blockValues[p] |= (part << (i * beta));
+ }
+ // 跳过这段 bitpacked 的位段，回到下一个字节边界
+ encode_pos = (bitStart + remainder * bw_i + 7) / 8;
+ } else {
+ // RLE: 先读 runCount (2 bytes)，随后是 run_length[]（bw 位）和 rle_values[]（bw_i 位）
+ int runCount = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ // 读取 run_length（每项 bw 位）
+ int[] run_length = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, runCount, run_length);
+
+ // 读取 rle_values（每项 bw_i 位）
+ int[] rle_values = new int[runCount];
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw_i, runCount, rle_values);
+
+ // 根据 run_lengths 展开并赋值
+ int pos = 0;
+ for (int r = 0; r < runCount && pos < remainder; r++) {
+ int len = run_length[r];
+ int val = rle_values[r];
+ for (int t = 0; t < len && pos < remainder; t++, pos++) {
+ blockValues[pos] |= (val << (i * beta));
+ }
+ }
+ // 注意：encode_pos 已由 decodeBitPacking 更新
+ }
+ }
+
+ // 7) 把 min_delta 加回每个位置
+ for (int p = 0; p < remainder; p++) {
+ blockValues[p] += min_delta;
+ }
+
+ return new DecodedBlock(blockValues, encode_pos);
+ }
+
+ /**
+ * 完整解码整列（逐块调用 decodeBlockValues）
+ *
+ * @param encoded_result 编码后字节数组（包含前 8 字节 header: data_length, block_size）
+ * @return 解码后的整列 int[]，长度 = data_length
+ */
+ public static int[] decodeColumnFully(byte[] encoded_result) {
+ int encode_pos = 0;
+ int data_length = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+ int block_size = readInt32BE(encoded_result, encode_pos);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+ int[] out = new int[data_length];
+
+ // 完整块
+ for (int b = 0; b < num_blocks; b++) {
+ DecodedBlock db = decodeBlockValues(encoded_result, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, b * block_size, block_size);
+ }
+
+ // 最后一块（如果有剩余）
+ if (remainder > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result, num_blocks, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ System.arraycopy(db.values, 0, out, num_blocks * block_size, remainder);
+ }
+
+ return out;
+ }
+
+ /**
+ * 严格 EM-parallel: 彻底解码两列为 int[]，然后逐行判断两个谓词同时成立的位置。
+ *
+ * @param encoded_result1 列1的编码字节数组
+ * @param encoded_result2 列2的编码字节数组
+ * @param upper_bound1 列1的上界谓词（< upper_bound1）
+ * @param upper_bound2 列2的上界谓词（< upper_bound2）
+ * @param result 用于输出匹配位置的数组（全表偏移 / 行号）
+ * @param result_length 长度容器（长度为1的数组，写回匹配数）
+ */
+ public static void QueryTwoColumnsStrictEMParallel(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 完整解码两列
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int[] col2 = decodeColumnFully(encoded_result2);
+
+ // 确定行数（取两列最小）
+ int n = Math.min(col1.length, col2.length);
+ result_length[0] = 0;
+
+ for (int i = 0; i < n; i++) {
+ if (col1[i] < upper_bound1 && col2[i] < upper_bound2) {
+ result[result_length[0]] = i;
+ result_length[0]++;
+ }
+ }
+ }
+
+ /**
+ * 严格 EM-pipelined: 完整解码第一列（物化为 values），根据第一列筛出 candidate positions（按块组织），
+ * 然后对第二列按块按需解码（只解包含 candidate 的块），在这些块中对 candidate 的局部索引做精确判定。
+ *
+ * @param encoded_result1 列1编码
+ * @param encoded_result2 列2编码
+ * @param upper_bound1
+ * @param upper_bound2
+ * @param result
+ * @param result_length
+ */
+ public static void QueryTwoColumnsStrictEMPipelined(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2, int[] result, int[] result_length) {
+
+ // 1) 解码第 1 列（完整解码以物化 tuple 的该属性）
+ int[] col1 = decodeColumnFully(encoded_result1);
+ int data_length = col1.length;
+
+ // 2) 构建候选位置分块索引（与 QueryWithIndices 中的策略一致）
+ int block_size = readInt32BE(encoded_result2, 4); // 注意：编码头部：前4字节 data_length，接着 4 字节 block_size
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ // 统计每个块中 candidate 数量
+ int[] blockCount = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ blockCount[bidx]++;
+ }
+ }
+
+ // 若没有 candidate，快速返回
+ int totalCandidates = 0;
+ for (int c : blockCount) totalCandidates += c;
+ result_length[0] = 0;
+ if (totalCandidates == 0) return;
+
+ // 为每块分配数组以记录 local indices
+ int[][] blockIndices = new int[num_blocks + 1][];
+ for (int b = 0; b <= num_blocks; b++) {
+ blockIndices[b] = new int[blockCount[b]];
+ }
+ int[] cursor = new int[num_blocks + 1];
+ for (int i = 0; i < data_length; i++) {
+ if (col1[i] < upper_bound1) {
+ int bidx = i / block_size;
+ int local = i % block_size;
+ blockIndices[bidx][cursor[bidx]++] = local;
+ }
+ }
+
+ // 3) 逐块扫描第二列：若该块没有 candidate -> 跳过（SkipBlock）；否则解码该块并测试
+ int encode_pos = 0;
+ int data_len_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ int bs_from_header = readInt32BE(encoded_result2, encode_pos);
+ encode_pos += 4;
+ // sanity check bs_from_header == block_size
+ // 逐块循环
+ for (int b = 0; b < num_blocks; b++) {
+ if (blockCount[b] == 0) {
+ // 跳过此块
+ encode_pos = SkipBlock(encoded_result2, b, block_size, block_size, encode_pos);
+ continue;
+ }
+ // 需要解码此块
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, block_size, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ // 对该块的候选局部索引测试第二列谓词
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ }
+
+ // 最后可能的不完整块
+ if (remainder > 0) {
+ int b = num_blocks;
+ if (blockCount[b] > 0) {
+ DecodedBlock db = decodeBlockValues(encoded_result2, b, block_size, remainder, encode_pos);
+ encode_pos = db.newEncodePos;
+ int[] vals = db.values;
+ for (int j = 0; j < blockCount[b]; j++) {
+ int localIdx = blockIndices[b][j];
+ if (localIdx >= 0 && localIdx < vals.length) {
+ if (vals[localIdx] < upper_bound2) {
+ int globalPos = b * block_size + localIdx;
+ result[result_length[0]] = globalPos;
+ result_length[0]++;
+ }
+ }
+ }
+ } else {
+ // 无 candidate，跳过或不处理
+ }
+ }
+ }
+
+ public static double computeSelectivity(long len1_0, long len2_0, long halfSize, long match) {
+ double sA = Double.NaN, sB = Double.NaN, pAB = Double.NaN, lift = Double.NaN;
+
+ if (halfSize <= 0) return lift;
+
+ // 基本概率
+ sA = (double) len1_0 / (double) halfSize;
+ sB = (double) len2_0 / (double) halfSize;
+ pAB = (double) match / (double) halfSize;
+
+ // lift = P(A∧B) / (P(A) P(B))，仅在分母非零时计算
+ if (sA > 0.0 && sB > 0.0) {
+ lift = pAB / (sA * sB);
+ }
+
+ return lift;
+ }
+ public static double phiCoefficient(long len1_0, long len2_0, long halfSize, long match) {
+ // 2x2 表格元素
+ double a = (double) match; // A ∧ B
+ double b = (double) (len1_0 - match); // A ∧ ¬B
+ double c = (double) (len2_0 - match); // ¬A ∧ B
+ double d = (double) (halfSize - (match + (len1_0 - match) + (len2_0 - match)));
+ // 等价于: d = halfSize - (a + b + c)
+
+ // 如果任何分量为负，输入可能不合法，返回 NaN
+ if (a < 0 || b < 0 || c < 0 || d < 0) {
+ return Double.NaN;
+ }
+
+ double numerator = a * d - b * c;
+ double denomTerm1 = (a + b) * (c + d);
+ double denomTerm2 = (a + c) * (b + d);
+
+ // 分母为 sqrt( denomTerm1 * denomTerm2 )
+ double denomProduct = denomTerm1 * denomTerm2;
+ if (denomProduct <= 0.0) {
+ return Double.NaN; // 避免除零或根号负数
+ }
+
+ double phi = numerator / Math.sqrt(denomProduct);
+ return phi;
+ }
+ // 放在类的末尾，作为一个新的测试 / helper
+ @Test
+ public void compareMaterializationStrategies() throws IOException {
+ // --- 基本设置，复用你 testQuery 中的路径 / 数据准备逻辑 ---
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/materialization/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+
+// // 这里为了演示，仅处理单个 CSV 文件（你可以循环多个文件）
+// File directory = new File(input_parent_dir);
+// File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+// if (csvFiles == null || csvFiles.length == 0) {
+// System.out.println("No csv files found under " + input_parent_dir);
+// return;
+// }
+//
+// // 选第一个文件作为 demo
+// File file = csvFiles[0];
+// String datasetName = extractFileName(file.toString());
+// System.out.println("Dataset: " + datasetName);
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 500;
+ String outputPath = output_parent_dir + "subcolumn_filter.csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "LM-pipelined",
+ "LM-parallel",
+ "Points",
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // 读取列并构造两列（复用你已有的读取逻辑）
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(!queryRange.containsKey(datasetName))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> raw = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) continue;
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) max_decimal = cur_decimal;
+ raw.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = raw.size();
+ int halfSize = totalSize / 2;
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) col1_data[i] = (int) (raw.get(i) * max_mul);
+ for (int i = 0; i < halfSize; i++) col2_data[i] = (int) (raw.get(i + halfSize) * max_mul);
+
+ int block_size = 512; // 选择一个 block size 做比较
+// int repeatTime = 200;
+ byte[] encoded_result1 = new byte[col1_data.length * 4];
+ byte[] encoded_result2 = new byte[col2_data.length * 4];
+
+ // 编码（复用你的 Encoder）
+ int length1 = SubcolumnTest.Encoder(col1_data, block_size, encoded_result1);
+ int length2 = SubcolumnTest.Encoder(col2_data, block_size, encoded_result2);
+
+ int upper = queryRange.containsKey(datasetName) ? queryRange.get(datasetName) : Integer.MAX_VALUE;
+
+// System.out.println("Running repeats: " + repeatTime + " upper=" + upper);
+
+ // ---------- 1) LM-pipelined: your existing QueryTwoColumns ----------
+ long tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ QueryTwoColumns(encoded_result1, encoded_result2, upper, upper);
+ }
+ long tEnd = System.nanoTime();
+ long lmPipelinedTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-pipelined avg ns: " + lmPipelinedTime);
+
+ // ---------- 2) LM-parallel: Query both columns separately -> intersect positions ----------
+ // helper arrays reused
+ int[] res1 = new int[encoded_result1.length];
+ int[] len1 = new int[1];
+ int[] res2 = new int[encoded_result2.length];
+ int[] len2 = new int[1];
+
+ // warm run to avoid JIT one-time overhead bias
+ Query(encoded_result1, upper, res1, len1);
+ Query(encoded_result2, upper, res2, len2);
+
+ tStart = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ // run both queries (they are pure functions on encoded bytes)
+ CompletableFuture<Void> future1 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result1, upper, res1, len1);
+ });
+
+ CompletableFuture<Void> future2 = CompletableFuture.runAsync(() -> {
+ Query(encoded_result2, upper, res2, len2);
+ });
+
+ // 等待两个查询都完成
+ try {
+ CompletableFuture.allOf(future1, future2).get();
+ } catch (InterruptedException | ExecutionException e) {
+ e.printStackTrace();
+ // 处理异常，可能需要中断循环或采取其他措施
+ Thread.currentThread().interrupt(); // 重新设置中断状态
+ break;
+ }
+ // intersect result sets (they are arrays of positions)
+//System.out.println(len1[0]);
+// System.out.println(len2[0]);
+//// 并行设置bit
+// long[] bits1 = new long[(halfSize + 63) / 64];
+// long[] bits2 = new long[(halfSize + 63) / 64];
+//
+//// 设置bit
+// for (int i = 0; i < len1[0]; i++) {
+// int pos = res1[i];
+// bits1[pos >> 6] |= (1L << (pos & 0x3F));
+// }
+//
+// for (int i = 0; i < len2[0]; i++) {
+// int pos = res2[i];
+// bits2[pos >> 6] |= (1L << (pos & 0x3F));
+// }
+//
+//// 求交集并计数
+// int match = 0;
+// for (int i = 0; i < bits1.length; i++) {
+// long intersection = bits1[i] & bits2[i];
+// match += Long.bitCount(intersection);
+// }
+// System.out.println(computeSelectivity(len1[0],len2[0],halfSize,match));
+
+ }
+ tEnd = System.nanoTime();
+ long lmParallelTime = (tEnd - tStart) / repeatTime;
+ System.out.println("LM-parallel avg ns: " + lmParallelTime);
+
+
+ int[] result = new int[encoded_result1.length];
+ int[] resultLen = new int[1];
+
+
+
+ // 最后打印一行小结
+// System.out.println("Summary (ns avg per query): LM-pipelined=" + lmPipelinedTime
+// + " LM-parallel=" + lmParallelTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(lmPipelinedTime),
+ String.valueOf(lmParallelTime),
+ String.valueOf(totalSize)
+ };
+ writer.writeRecord(record);
+
+ }
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaxWithNULLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaxWithNULLTest.java
new file mode 100644
index 0000000..f6375668
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnMaxWithNULLTest.java
@@ -0,0 +1,1157 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SubcolumnMaxWithNULLTest {
+
+ public static int bitWidth(int value) {
+ if(value == 0) return 1;
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+
+ public static void Query(byte[] encoded_result) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQueryMax(encoded_result, i, block_size, block_size, encode_pos, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ result[result_length[0]] = value;
+ result_length[0]++;
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockQueryMax(encoded_result, num_blocks, block_size, remainder, encode_pos,
+ result, result_length);
+ }
+
+ // for (int i = 0; i < result_length[0]; i++) {
+ // System.out.print(result[i] + " ");
+ // }
+ // System.out.println();
+
+ }
+
+ public static int BlockQueryMax(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ // System.out.println("m: " + m);
+
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ result[result_length[0]] = min_delta[0];
+ result_length[0]++;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (candidate_length == 1) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int maxPart = 0;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+ int value = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos * 8 + index * bitWidthList[i], bitWidthList[i]);
+
+ if (value > maxPart) {
+ maxPart = value;
+
+ new_length = 0;
+ candidate_indices[new_length] = index;
+ new_length++;
+ } else if (value == maxPart) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+
+ // for (int j = 0; j < candidate_length; j++) {
+ // int index = candidate_indices[j];
+ // if (subcolumnList[i][index] == maxPart) {
+ // candidate_indices[new_length] = index;
+ // new_length++;
+ // }
+ // }
+
+ candidate_length = new_length;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ if (candidate_length == 1) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int maxPart = 0;
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] > maxPart) {
+ maxPart = rle_values[rleIndex];
+
+ new_length = 0;
+ } else if (rle_values[rleIndex] == maxPart) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ // for (int j = 0; j < candidate_length; j++) {
+ // int index_candidate = candidate_indices[j];
+
+ // while (rleIndex < index && currentPos + run_length[rleIndex] <=
+ // index_candidate) {
+ // currentPos += run_length[rleIndex];
+ // rleIndex++;
+ // }
+
+ // if (rleIndex < index) {
+ // if (rle_values[rleIndex] == maxPart) {
+ // candidate_indices[new_length] = index_candidate;
+ // new_length++;
+ // }
+ // }
+ // }
+
+ candidate_length = new_length;
+ }
+ }
+
+ result[result_length[0]] = candidate_indices[0];
+ result_length[0]++;
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
+ // String output_parent_dir = parent_dir + "result/query_vs_block/";
+
+ int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (int block_size : block_size_list) {
+ String outputPath = output_parent_dir + "subcolumn_query_max_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("block_size: " + block_size);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ @Test
+ public void testQueryBeta() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/query_max_null/"; //""D:/encoding-subcolumn/result/";
+
+
+// String outputPath = output_parent_dir + "max_with_null.csv";
+
+
+// int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+// 24, 25, 26, 27, 28, 29, 30, 31 };
+ double[] null_rate_list = {0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1};
+// double[] null_rate_list = {0.3,0.4,0.5,0.6,0.7,0.8,0.9};
+ int block_size = 512;
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (double null_rate : null_rate_list) {
+ System.out.println("null_rate: "+null_rate);
+ String outputPath = output_parent_dir + "subcolumn_query_max_null_" + null_rate + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int nullCountPerBlock = (int) (null_rate * (double) block_size); // 每块中要设置为null的数量
+ int new_arr_length = (int) ((double)(data1.size()/block_size*block_size)*(1-null_rate)+
+ (double)(data1.size()-data1.size()/block_size*block_size)*(1-null_rate));
+ System.out.println("new_arr_length:"+new_arr_length);
+ int[] data2_arr_new = new int[new_arr_length];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+
+
+ java.util.BitSet bitmap = new java.util.BitSet(data1.size());
+ int new_array_index = 0;
+
+ if(null_rate==1){
+ int[] bitmap_bit = new int[data1.size()];
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ for(int i=0;i<data1.size();i++){
+ bitmap_bit[i] = 0;
+ }
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+ double ratioTmp = 0;
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ String[] decode_values = new String[data1.size()];
+ for(int i=0;i<data1.size();i++){
+ decode_values[i] = "NULL";
+ }
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ continue;
+ } else if(null_rate != 0 && null_rate != 1){
+ for (int blockStart = 0; blockStart < data1.size(); blockStart += block_size) {
+ int blockEnd = Math.min(blockStart + block_size, data1.size());
+ int actualBlockSize = (int) ((double)(blockEnd - blockStart)*(1-null_rate));
+ int actualNullCount = Math.min(nullCountPerBlock, actualBlockSize);
+
+ // 创建当前块的索引列表用于随机选择
+ java.util.List<Integer> indices = new java.util.ArrayList<>();
+ for (int i = blockStart; i < blockEnd; i++) {
+ indices.add(i);
+ }
+
+ // 随机打乱并选择要设置为null的位置
+ java.util.Collections.shuffle(indices);
+ java.util.List<Integer> selectedIndices = new java.util.ArrayList<>();
+ for (int i = 0; i < actualNullCount; i++) {
+ selectedIndices.add(indices.get(i));
+ }
+ java.util.Collections.sort(selectedIndices);
+ int i = 0;
+ int j = blockStart;
+// int nullIndex;
+ while (j < blockEnd) {
+ if (i < actualNullCount && j == selectedIndices.get(i)) {
+ // 这个位置被移除，设置bitmap并跳过
+ bitmap.set(j);
+ i++;
+ } else {
+ // 这个位置保留，复制到新数组
+// if(new_array_index==70000){
+// System.out.println(j/block_size*block_size);
+// System.out.println(j);
+// System.out.println(new_array_index);
+// }
+ data2_arr_new[new_array_index] = data2_arr[j];
+ new_array_index++;
+ if(new_array_index == new_arr_length) break;
+ }
+ j++;
+ }
+ if(new_array_index == new_arr_length) break;
+ }
+ }else {
+ data2_arr_new = data2_arr;
+ }
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr_new, (block_size-nullCountPerBlock), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ SubcolumnMaxWithNULLTest.Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnNoPruneBetaTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnNoPruneBetaTest.java
new file mode 100644
index 0000000..1fa50c2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnNoPruneBetaTest.java
@@ -0,0 +1,936 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Stream;
+
+public class SubcolumnNoPruneBetaTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int cost1 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+
+ int count = 1;
+
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+
+ for (int j = 1; j < x_length; j++) {
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+ }
+
+ }
+
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+
+ cost1 += Math.min(bpe_cost_single[i], Math.min(rle_cost_single[i], de_cost_single[i]));
+ }
+
+ int cMin = cost1;
+
+ int[] beta_list = new int[m - 1];
+ for (int i = 0; i < m - 1; i++) {
+ beta_list[i] = i + 2;
+ }
+
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+
+ // System.out.println("subcolumn index: " + i);
+
+ int bpCost = bitWidthListList[i] * x_length;
+
+ // System.out.println("bpCost: " + bpCost);
+
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+ // uniqueValues.add(currentNumber);
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = beta * index + bitWidth(x_length) * index;
+
+ // System.out.println("rleCost: " + rleCost);
+ //
+ int deCost = 0;
+
+ Set<Integer> uniqueValues = new HashSet<>();
+
+ for (int j = 0; j < x_length; j++) {
+ uniqueValues.add(subcolumnList[i][j]);
+ }
+
+ deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+
+ // System.out.println("deCost: " + deCost);
+
+ int cost_min = Math.min(bpCost, Math.min(rleCost, deCost));
+
+ // System.out.println("cost_min: " + cost_min);
+
+ cost += cost_min;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ // System.out.println("betaBest: " + betaBest);
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ // System.out.println("maxValue: " + maxValue);
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ // System.out.println("All zero list.");
+ return encode_pos;
+ }
+
+ int l;
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+ }
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ index++;
+
+ rleCost = bitWidth(block_size) * index + beta[0] * index;
+
+ int dict_bit_width = bitWidth(cardinality);
+ int dicCost = dict_bit_width * list_length + cardinality * (beta[0]);
+ if (dicCost < rleCost && dicCost < bpCost) {
+ encodingType[i] = 2;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ }
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bitWidth(block_size), encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if (type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width,
+ // cardinality, dict_value_list);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length,
+ subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // if (block_index % 16 == 0) {
+ // if (block_index % 32 == 0) {
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ // beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int beta_value) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = beta_value;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test1() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/compression_vs_beta_noprune/";
+
+ File outputDir = new File(output_parent_dir);
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ }
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int block_size = 1024;
+
+ int repeatTime = 500;
+
+ List<String> datasetList = new ArrayList<>();
+ datasetList.add("Arade4");
+ datasetList.add("Bird-migration");
+ datasetList.add("Bitcoin-price");
+ datasetList.add("City-temp");
+ datasetList.add("Dewpoint-temp");
+ datasetList.add("EPM-Education");
+ datasetList.add("Gov10");
+ datasetList.add("POI-lat");
+ datasetList.add("IR-bio-temp");
+ datasetList.add("PM10-dust");
+ datasetList.add("Stocks-DE");
+ datasetList.add("Stocks-UK");
+ datasetList.add("Stocks-USA");
+ datasetList.add("Wind-Speed");
+ datasetList.add("Wine-Tasting");
+
+ for (int beta : beta_list) {
+
+ String outputPath = output_parent_dir + "subcolumn_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ for (String datasetName : datasetList) {
+ String filePath = input_parent_dir + datasetName + ".csv";
+ File file = new File(filePath);
+
+ if (!file.exists()) {
+ System.out.println("File not found: " + filePath);
+ continue;
+ }
+
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnValuesTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnValuesTest.java
new file mode 100644
index 0000000..e7b6be3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnValuesTest.java
@@ -0,0 +1,973 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnOnValuesTest {
+
+ public static int bitWidth(int value) {
+ if (value == 0)
+ return 1;
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ /**
+ * Bit width for one signed int in a block: non-negative values use leading-zero
+ * trimmed width; negative values use full 32 bits (two's complement).
+ */
+ public static int valueBitWidth(int value) {
+ if (value < 0) {
+ return 32;
+ }
+ return bitWidth(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int cost0 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+
+ int count = 1;
+
+ de_cost_single[i] = 1;
+
+ for (int j = 0; j < x_length; j++) {
+
+ // if (count * (1 + (int) Math.ceil(Math.log(x_length))) >= x_length) {
+ // rle_cost_single[i] = x_length + 1;
+ // break;
+ // }
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length + 2 * (1 + 1);
+ }
+
+ }
+
+ rle_cost_single[i] = count * (1 + (int) Math.ceil(Math.log(x_length)));
+
+ cost0 += Math.min(bpe_cost_single[i], Math.min(rle_cost_single[i], de_cost_single[i]));
+ }
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ // int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ // for (int i = 0; i < l; i++) {
+ // int maxValuePart = 0;
+ // for (int j = 0; j < x_length; j++) {
+ // subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][j] > maxValuePart) {
+ // maxValuePart = subcolumnList[i][j];
+ // }
+ // }
+ // bitWidthListList[i] = bitWidth(maxValuePart);
+ // }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+
+ // int bpCost = bpe_cost_single[i * beta] * beta;
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start - 1 >= i * beta && bpe_cost_single[beta_start - 1] == 0) {
+ beta_start--;
+ }
+
+ int bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
+ int rleCost = 0;
+
+ // int lowestBitIndex = 0;
+ // int currentLowestBit = subcolumnList[i][0] & 1;
+
+ // for (int j = 1; j < x_length; j++) {
+ // int lowestBit = subcolumnList[i][j] & 1; // 获取当前元素的最低位
+ // if (lowestBit != currentLowestBit) {
+ // lowestBitIndex++;
+ // currentLowestBit = lowestBit;
+ // }
+ // }
+
+ // if (bw * lowestBitIndex + bitWidthListList[i] * lowestBitIndex >= bpCost) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+ int currentNumber = (x[0] >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 1; j < x_length; j++) {
+ int currentNumber_j = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (currentNumber_j != currentNumber) {
+ index++;
+ currentNumber = currentNumber_j;
+ }
+ if (bw * index + bitWidth(x_length) * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidth(x_length) * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int m = 1;
+ for (int k : list) {
+ int w = valueBitWidth(k);
+ if (w > m) {
+ m = w;
+ }
+ }
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) * 2 / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ // uniqueValues.add(previous);
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ // if(currentNumber == 6){
+ // System.out.println("currentNumber == 6 && i==0");
+ // System.out.println(uniqueValues);
+ // }
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (cardinality < Math.pow(2, bitWidthList[i] - 1)) {
+ // test dictionary encoding
+ int dict_bit_width = bitWidth(cardinality);
+ int dicCost = dict_bit_width * list_length + cardinality * (bitWidthList[i] + dict_bit_width);
+ if (dicCost < rleCost && dicCost < bpCost) {
+ // if dictionary encoding
+ // int dict_bit_width = bitWidth(cardinality) ;
+ encodingType[i] = 2;
+
+ // System.out.println(uniqueValues);
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ // int[] dict_value_list = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ // dict_value_list[j] = j;
+ }
+ // System.out.println(valueToCode);
+ // System.out.println(list_length);
+ // System.out.println(beta[0]);
+ // System.out.println(Arrays.toString(subcolumnList[i]));
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+ // encode_pos = bitPacking(dict_value_list, dict_bit_width, encode_pos,
+ // encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if (type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width,
+ // cardinality, dict_value_list);
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length,
+ subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] data_block = new int[remainder];
+ int base = block_index * block_size;
+ for (int j = 0; j < remainder; j++) {
+ data_block[j] = data[base + j];
+ }
+
+ if (block_index == 0) {
+ int m = 1;
+ for (int j = 0; j < remainder; j++) {
+ int w = valueBitWidth(data_block[j]);
+ if (w > m) {
+ m = w;
+ }
+ }
+
+ beta[0] = Subcolumn(data_block, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_block, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; // "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/"; // ""D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_dictionary_on_values.csv";
+
+ // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ // if(! datasetName.equals("Stocks-UK")){
+ // continue;
+ // }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Dictionary)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnlyDictionaryTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnlyDictionaryTest.java
new file mode 100644
index 0000000..c0eb8d1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnOnlyDictionaryTest.java
@@ -0,0 +1,636 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+public class SubcolumnOnlyDictionaryTest {
+
+ public static int bitWidth(int value) {
+ if(value==0) return 1;
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ // Subcolumn 方法简化，只返回固定的 beta 值
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+ // 对于纯字典编码，我们可以使用固定的 beta 值
+ // 或者根据数据特征选择一个合适的值
+ int[] beta_list = { 2, 3, 4 };
+
+ // 简单返回第一个可用的 beta
+ for (int beta : beta_list) {
+ if (beta <= m) {
+ return beta;
+ }
+ }
+ return 2; // 默认返回 2
+ }
+
+ /**
+ * 纯字典编码的编码器
+ * 对每个分列都使用字典编码
+ */
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ // 存储 m 值
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+ int[][] subcolumnList = new int[l][list_length];
+
+ // 存储 beta 值
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int mask = (1 << beta[0]) - 1;
+
+ // 分解成子列
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ // 存储每个分列的位宽
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ // 对每个分列进行字典编码
+ for (int i = l - 1; i >= 0; i--) {
+ // 获取唯一值
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ uniqueValues.add(subcolumnList[i][j]);
+ }
+ int cardinality = uniqueValues.size();
+
+ // 构建字典
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ dict_value_list[j] = j;
+ }
+
+ // 编码数据
+ int[] encodedData = new int[list_length];
+ for (int j = 0; j < list_length; j++) {
+ encodedData[j] = valueToCode.get(subcolumnList[i][j]);
+ }
+
+ // 存储基数
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ // 计算字典编码所需的位宽
+ int dict_bit_width = bitWidth(cardinality - 1);
+ if (dict_bit_width == 0) {
+ dict_bit_width = 1;
+ }
+
+ // 存储字典键（原始值）
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ // 存储字典值（编码）
+ encode_pos = bitPacking(dict_value_list, dict_bit_width, encode_pos, encoded_result, cardinality);
+
+ // 存储编码后的数据
+ encode_pos = bitPacking(encodedData, dict_bit_width, encode_pos, encoded_result, list_length);
+ }
+
+ return encode_pos;
+ }
+
+ /**
+ * 纯字典编码的解码器
+ */
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ // 解码每个分列的字典编码
+ for (int i = l - 1; i >= 0; i--) {
+ // 读取基数
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ int dict_bit_width = bitWidth(cardinality - 1);
+ if (dict_bit_width == 0) {
+ dict_bit_width = 1;
+ }
+
+ // 解码字典
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ // 解码数据
+ int[] encodedData = new int[list_length];
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, encodedData);
+
+ // 构建解码映射
+ Map<Integer, Integer> codeToValue = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ codeToValue.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ // 解码数据
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = codeToValue.get(encodedData[j]);
+ }
+ }
+
+ // 重构原始数据
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[1];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size, remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos, encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int min_delta = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos, block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta;
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, encoded_result, beta);
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder, encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_only_dictionary.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime = (e - s) / repeatTime;
+
+ double compressed_size = length;
+ double ratio = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ System.out.println(ratio);
+
+ s = System.nanoTime();
+ int[] data2_arr_decoded = null;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+ e = System.nanoTime();
+ decodeTime = (e - s) / repeatTime;
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Only Dictionary)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryBPTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryBPTest.java
new file mode 100644
index 0000000..54cfefc
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryBPTest.java
@@ -0,0 +1,988 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+public class SubcolumnPointQueryBPTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnBP(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ // int rleCost = 0;
+
+ // // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+
+ // int index = 0;
+
+ // boolean bpBest = false;
+
+ // for (int j = 1; j < x_length; j++) {
+ // if (subcolumnList[i][j] != currentNumber) {
+ // index++;
+ // currentNumber = subcolumnList[i][j];
+ // }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ // }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // index++;
+
+ // System.out.println("index: " + index);
+
+ // rleCost = bw * index + bitWidthListList[i] * index;
+
+ // // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += bpCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // int bpCost = bitWidthList[i] * list_length;
+ // int rleCost = 0;
+
+ // int previous = subcolumnList[i][0];
+ // int index = 0;
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // index++;
+ // previous = currentNumber;
+ // }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ // }
+
+ // index++;
+
+ // rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ // encodingType[i] = 1;
+
+ // encoded_result[encode_pos] = (byte) (index >> 8);
+ // encode_pos += 1;
+ // encoded_result[encode_pos] = (byte) (index & 0xFF);
+ // encode_pos += 1;
+
+ // index = 0;
+ // int[] run_length = new int[list_length];
+ // int[] rle_values = new int[list_length];
+ // previous = subcolumnList[i][0];
+
+ // for (int j = 1; j < list_length; j++) {
+ // int currentNumber = subcolumnList[i][j];
+ // if (currentNumber != previous) {
+ // run_length[index] = j;
+ // rle_values[index] = previous;
+ // index++;
+ // previous = currentNumber;
+ // }
+ // }
+
+ // run_length[index] = list_length;
+ // rle_values[index] = previous;
+ // index++;
+
+ // encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ // encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ // if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ // } else {
+ // int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ // encode_pos += 2;
+
+ // int[] run_length = new int[index];
+ // int[] rle_values = new int[index];
+
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ // int currentIndex = 0;
+ // for (int j = 0; j < index; j++) {
+ // int endPos = run_length[j];
+ // int value = rle_values[j];
+ // while (currentIndex < endPos) {
+ // subcolumnList[i][currentIndex] = value;
+ // currentIndex++;
+ // }
+ // }
+ // }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+//
+// if (block_index == 0) {
+// int maxValue = 0;
+// for (int j = 0; j < remainder; j++) {
+// if (data_delta[j] > maxValue) {
+// maxValue = data_delta[j];
+// }
+// }
+// int m = bitWidth(maxValue);
+//
+// beta[0] = SubcolumnBP(data_delta, remainder, m, block_size);
+// }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+// beta[0] = 2;
+
+ int pre_encode_pos = encode_pos;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ int[] result = new int[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+ System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, int[] result) {
+
+ encode_pos += 2;
+
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+ System.out.println("m: "+m);
+ System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ int result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+// int type = encodingType[i];
+// if (type == 0) {
+ encode_pos *= 8;
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+// } else {
+//
+// int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+//
+// encode_pos += 2;
+//
+//
+// int[] run_length = new int[index];
+// int[] rle_values = new int[index];
+//
+// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+// encode_pos *= 8;
+// int accumulate_run_length = 0;
+// if(pos_in_cur_block == accumulate_run_length){
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos, bitWidthList[i]);
+// }else{
+// for(int x=0;x<index;x++){
+// accumulate_run_length += run_length[x];
+// if(pos_in_cur_block < accumulate_run_length ){
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos + x * bitWidthList[i], bitWidthList[i]);
+// break;
+// }
+// }
+// }
+// encode_pos += index * bitWidthList[i];
+// encode_pos = (encode_pos + 7) / 8;
+//
+//// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+//// rle_values);
+//
+//
+// }
+ result_value <<= beta;
+ }
+ result[0]= result_value;
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+// // String output_parent_dir = parent_dir + "result/";
+//
+// // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "subcolumn_bp.csv";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query_bp/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+ int[] beta_list = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31};
+ // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+ for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")||datasetName.equals("Gov10")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ Random random = new Random();
+
+ // 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data1.size()/block_size*block_size;
+
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int randomNumber = random.nextInt(max_random_value);
+ Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryRLETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryRLETest.java
new file mode 100644
index 0000000..8b40625
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryRLETest.java
@@ -0,0 +1,987 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Random;
+
+public class SubcolumnPointQueryRLETest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnRLE(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += rleCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ // int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ // encodingType[i] = 0;
+
+ // encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = SubcolumnRLE(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int beta_value) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = beta_value;
+
+ int pre_encode_pos = encode_pos;
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ int[] result = new int[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+ System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, int[] result) {
+
+ encode_pos += 2;
+
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+ System.out.println("m: "+m);
+ System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ int result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+// int type = encodingType[i];
+// if (type == 0) {
+// encode_pos *= 8;
+// result_value += SubcolumnTest.bytesToInt(encoded_result,
+// encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+//
+// encode_pos += remainder * bitWidthList[i];
+// encode_pos = (encode_pos + 7) / 8;
+
+// } else {
+//
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos *= 8;
+ int accumulate_run_length = 0;
+ if(pos_in_cur_block == accumulate_run_length){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos, bitWidthList[i]);
+ }else{
+ for(int x=0;x<index;x++){
+ accumulate_run_length += run_length[x];
+ if(pos_in_cur_block < accumulate_run_length ){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + x * bitWidthList[i], bitWidthList[i]);
+ break;
+ }
+ }
+ }
+ encode_pos += index * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+//// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+//// rle_values);
+//
+//
+// }
+ result_value <<= beta;
+ }
+ result[0]= result_value + min_delta[0];
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/";
+// // String output_parent_dir = parent_dir + "result/";
+//
+// // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "subcolumn_rle.csv";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query_rle/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+ int[] beta_list = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31}; // int block_size = 512;
+ int block_size = 512;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+ for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")||datasetName.equals("Gov10")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 10];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ Random random = new Random();
+// 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data1.size()/block_size*block_size;
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int randomNumber = random.nextInt(max_random_value);
+ Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryTest.java
new file mode 100644
index 0000000..677f6ad
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPointQueryTest.java
@@ -0,0 +1,445 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+public class SubcolumnPointQueryTest {
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ encoded_result[encode_pos] = (byte) (min_delta[0] >> 24);
+ encoded_result[encode_pos + 1] = (byte) (min_delta[0] >> 16);
+ encoded_result[encode_pos + 2] = (byte) (min_delta[0] >> 8);
+ encoded_result[encode_pos + 3] = (byte) min_delta[0];
+ encode_pos += 4;
+
+ encode_pos = SubcolumnTest.SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int beta_value) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = beta_value;
+// System.out.println(Arrays.toString(beta));
+
+ int pre_encode_pos = encode_pos;
+
+// encode_pos += 2;
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos += 2;
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ pre_encode_pos = encode_pos;
+ }
+
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ encoded_result[encode_pos] = (byte) (value >> 24);
+ encoded_result[encode_pos + 1] = (byte) (value >> 16);
+ encoded_result[encode_pos + 2] = (byte) (value >> 8);
+ encoded_result[encode_pos + 3] = (byte) value;
+ encode_pos += 4;
+ }
+ encoded_result[pre_encode_pos+1] = (byte) (remainder*4);
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ int encode_block_length = encode_pos-pre_encode_pos;
+ encoded_result[pre_encode_pos] = (byte) (encode_block_length >> 8);
+ encoded_result[pre_encode_pos+1] = (byte) encode_block_length;
+ }
+
+ return encode_pos;
+ }
+
+
+ public static void Query(byte[] encoded_result, int point) {
+
+ int encode_pos = 0;
+
+// int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+// |
+// ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int number_of_skipped_blocks = point / block_size;
+ int acc_add_encode_pos = 0;
+ for (int i = 0; i < number_of_skipped_blocks; i++) {
+ int tmp_acc_add_encode_pos = (((encoded_result[encode_pos+acc_add_encode_pos] & 0xFF) << 8)|(encoded_result[encode_pos+acc_add_encode_pos+1] & 0xFF));
+ acc_add_encode_pos += tmp_acc_add_encode_pos;
+ }
+ encode_pos += acc_add_encode_pos;
+ number_of_skipped_blocks += 1;
+ // 在当前的block中的位置
+ int pos_in_cur_block = point % block_size;
+ int[] result = new int[1];
+ encode_pos = BlockQueryIndex(encoded_result, number_of_skipped_blocks, block_size,
+ block_size, encode_pos, pos_in_cur_block, result);
+ System.out.println("result:" + result[0]);
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int pos_in_cur_block, int[] result) {
+
+ encode_pos += 2;
+
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+
+// // 候选索引列表，当前分列值和 lower_bound 相应值相等的索引
+// int[] candidate_indices = new int[remainder];
+// int candidate_length = 0;
+// for (int i = 0; i < remainder; i++) {
+// candidate_indices[i] = i;
+// candidate_length++;
+// }
+
+ if (m == 0) {
+ result[0] = 0;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+ System.out.println("m: "+m);
+ System.out.println("beta: "+beta);
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+ int result_value = 0;
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ encode_pos *= 8;
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + pos_in_cur_block * bitWidthList[i], bitWidthList[i]);
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos *= 8;
+ int accumulate_run_length = 0;
+ if(pos_in_cur_block == accumulate_run_length){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos, bitWidthList[i]);
+ }else{
+ for(int x=0;x<index;x++){
+ accumulate_run_length += run_length[x];
+ if(pos_in_cur_block < accumulate_run_length ){
+ result_value += SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + x * bitWidthList[i], bitWidthList[i]);
+ break;
+ }
+ }
+ }
+ encode_pos += index * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+// encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+// rle_values);
+
+
+ }
+ result_value <<= beta;
+ }
+ result[0]= result_value;
+
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ @Test
+ public void testQueryBeta() throws IOException {
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/point_query/";//"D:/encoding-subcolumn/result/query_vs_beta/";
+
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 500;
+
+// repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (int beta : beta_list) {
+ String outputPath = output_parent_dir + "subcolumn_point_query_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")||datasetName.equals("POI-lat")) continue;
+// if(!datasetName.equals("Bitcoin-price")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ Random random = new Random();
+
+ // 生成 [0, length-1] 范围内的随机整数
+
+ int max_random_value = data1.size()/block_size*block_size;
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int randomNumber = random.nextInt(max_random_value);
+ SubcolumnPointQueryTest.Query(encoded_result, randomNumber);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTest.java
new file mode 100644
index 0000000..f430a8a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTest.java
@@ -0,0 +1,773 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+
+public class SubcolumnPrune2NewTest {
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(int[] values, int length, int shiftAmount,
+ int mask, int limit) {
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnValue = (x[j] >> i) & 1;
+ if (subcolumnValue == 1) {
+ hasOne = true;
+ }
+ if (subcolumnValue != currentValue) {
+ runCount++;
+ currentValue = subcolumnValue;
+ changed = true;
+ }
+ }
+
+ bpeCostSingle[i] = hasOne ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = changed ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ cost1 += rleCostSingle[i];
+ } else {
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ int rleCost = countGroupedRuns(x, xLength, groupStart, mask)
+ * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount = countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, threshold[beta - 1]);
+ if (distinctCount < threshold[beta - 1]) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize) {
+ int listLength = list.length;
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = new int[l];
+ int[] encodingType = new int[l];
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+ boolean[] seenValues = new boolean[mask + 1];
+ int[] dictKeyList = new int[mask + 1];
+ int[] codeMap = new int[mask + 1];
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+
+ int bpCost = bitWidthList[i] * listLength;
+ int previous = subcolumnBuffer[0];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runCount++;
+ int rleCost = bw * runCount + betaValue * runCount;
+
+ int cardinality = 0;
+ for (int value = 0; value <= mask; value++) {
+ seenValues[value] = false;
+ }
+ for (int j = 0; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (!seenValues[current]) {
+ seenValues[current] = true;
+ cardinality++;
+ }
+ }
+
+ int dictBitWidth = bitWidth(cardinality);
+ int dictCost = dictBitWidth * listLength
+ + cardinality * (bitWidthList[i] + dictBitWidth);
+
+ if (dictCost < rleCost && dictCost < bpCost) {
+ encodingType[i] = 2;
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if (seenValues[value]) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ continue;
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+ encodePos = bitPacking(subcolumnBuffer, bitWidthList[i], encodePos, encodedResult,
+ listLength);
+ } else {
+ encodingType[i] = 1;
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ int index = 0;
+ previous = subcolumnBuffer[0];
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runLength[index] = j;
+ rleValues[index] = previous;
+ index++;
+ previous = current;
+ }
+ }
+ runLength[index] = listLength;
+ rleValues[index] = previous;
+ index++;
+
+ encodePos = bitPacking(runLength, bw, encodePos, encodedResult, index);
+ encodePos = bitPacking(rleValues, bitWidthList[i], encodePos, encodedResult,
+ index);
+ }
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ int listLength = list.length;
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+
+ if (type == 0) {
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ listLength, subcolumnBuffer);
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j];
+ while (currentIndex < endPos) {
+ subcolumnBuffer[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = dictKeyList[subcolumnBuffer[j]];
+ }
+ }
+
+ int shiftAmount = i * beta;
+ for (int j = 0; j < listLength; j++) {
+ list[j] |= subcolumnBuffer[j] << shiftAmount;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ tsBlockDelta[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return tsBlockDelta;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ int[] minDelta = new int[1];
+ int[] dataDelta = getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ if (blockIndex == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+ beta[0] = Subcolumn(dataDelta, remainder, bitWidth(maxValue), blockSize);
+ }
+
+ return SubcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize);
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int[] blockData = new int[remainder];
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, blockData, blockSize);
+
+ int base = blockIndex * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = blockData[i] + minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult,
+ beta);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockEncoder(data, numBlocks, blockSize, remainder, encodePos,
+ encodedResult, beta);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTimeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTimeTest.java
new file mode 100644
index 0000000..f835918
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2NewTimeTest.java
@@ -0,0 +1,917 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class SubcolumnPrune2NewTimeTest {
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(int[] values, int length, int shiftAmount,
+ int mask, int limit) {
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnValue = (x[j] >> i) & 1;
+ if (subcolumnValue == 1) {
+ hasOne = true;
+ }
+ if (subcolumnValue != currentValue) {
+ runCount++;
+ currentValue = subcolumnValue;
+ changed = true;
+ }
+ }
+
+ bpeCostSingle[i] = hasOne ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = changed ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ cost1 += rleCostSingle[i];
+ } else {
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ int rleCost = countGroupedRuns(x, xLength, groupStart, mask)
+ * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount = countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, threshold[beta - 1]);
+ if (distinctCount < threshold[beta - 1]) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize) {
+ int listLength = list.length;
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = new int[l];
+ int[] encodingType = new int[l];
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+ boolean[] seenValues = new boolean[mask + 1];
+ int[] dictKeyList = new int[mask + 1];
+ int[] codeMap = new int[mask + 1];
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+
+ int bpCost = bitWidthList[i] * listLength;
+ int previous = subcolumnBuffer[0];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runCount++;
+ int rleCost = bw * runCount + betaValue * runCount;
+
+ int cardinality = 0;
+ for (int value = 0; value <= mask; value++) {
+ seenValues[value] = false;
+ }
+ for (int j = 0; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (!seenValues[current]) {
+ seenValues[current] = true;
+ cardinality++;
+ }
+ }
+
+ int dictBitWidth = bitWidth(cardinality);
+ int dictCost = dictBitWidth * listLength
+ + cardinality * (bitWidthList[i] + dictBitWidth);
+
+ if (dictCost < rleCost && dictCost < bpCost) {
+ encodingType[i] = 2;
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if (seenValues[value]) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ continue;
+ }
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+ encodePos = bitPacking(subcolumnBuffer, bitWidthList[i], encodePos, encodedResult,
+ listLength);
+ } else {
+ encodingType[i] = 1;
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ int index = 0;
+ previous = subcolumnBuffer[0];
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runLength[index] = j;
+ rleValues[index] = previous;
+ index++;
+ previous = current;
+ }
+ }
+ runLength[index] = listLength;
+ rleValues[index] = previous;
+ index++;
+
+ encodePos = bitPacking(runLength, bw, encodePos, encodedResult, index);
+ encodePos = bitPacking(rleValues, bitWidthList[i], encodePos, encodedResult,
+ index);
+ }
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ int listLength = list.length;
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+
+ if (type == 0) {
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ listLength, subcolumnBuffer);
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j];
+ while (currentIndex < endPos) {
+ subcolumnBuffer[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = dictKeyList[subcolumnBuffer[j]];
+ }
+ }
+
+ int shiftAmount = i * beta;
+ for (int j = 0; j < listLength; j++) {
+ list[j] |= subcolumnBuffer[j] << shiftAmount;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ tsBlockDelta[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return tsBlockDelta;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta, long[] betaTime,
+ long[] encodeTime) {
+ int[] minDelta = new int[1];
+ int[] dataDelta = getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ // if (blockIndex == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ long betaStart = System.nanoTime();
+ beta[0] = Subcolumn(dataDelta, remainder, bitWidth(maxValue), blockSize);
+ long betaEnd = System.nanoTime();
+ betaTime[0] += (betaEnd - betaStart);
+ // }
+
+ long encodeStart = System.nanoTime();
+ encodePos = SubcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize);
+ long encodeEnd = System.nanoTime();
+ encodeTime[0] += (encodeEnd - encodeStart);
+ return encodePos;
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int[] blockData = new int[remainder];
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, blockData, blockSize);
+
+ int base = blockIndex * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = blockData[i] + minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, long[] betaTime,
+ long[] encodeTime) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult,
+ beta, betaTime, encodeTime);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockEncoder(data, numBlocks, blockSize, remainder, encodePos,
+ encodedResult, beta, betaTime, encodeTime);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "subcolumn_dictionary3new.csv";
+
+ int blockSize = 512;
+ int repeatTime = 500;
+ repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Beta Selection Time",
+ "Subcolumn Encode Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ System.out.println(maxDecimal);
+ byte[] encodedResult = new byte[data2Arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressedSize = 0;
+ long betaTime = 0;
+ long subcolumnEncodeTime = 0;
+ int length = 0;
+ long[] betaTimeArr = new long[1];
+ long[] subcolumnEncodeTimeArr = new long[1];
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ betaTimeArr[0] = 0;
+ subcolumnEncodeTimeArr[0] = 0;
+ length = Encoder(data2Arr, blockSize, encodedResult, betaTimeArr,
+ subcolumnEncodeTimeArr);
+ betaTime += betaTimeArr[0];
+ subcolumnEncodeTime += subcolumnEncodeTimeArr[0];
+ }
+ long end = System.nanoTime();
+
+ encodeTime += ((end - start) / repeatTime);
+ compressedSize += length;
+ betaTime /= repeatTime;
+ subcolumnEncodeTime /= repeatTime;
+
+ double ratioTmp = compressedSize / (double) (data1.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ end = System.nanoTime();
+ decodeTime += ((end - start) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(betaTime),
+ String.valueOf(subcolumnEncodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2Test.java
new file mode 100644
index 0000000..a58bff6
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPrune2Test.java
@@ -0,0 +1,1094 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Stream;
+
+public class SubcolumnPrune2Test {
+
+ public static int bitWidth(int value) {
+ // if(value==0) return 1;
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+
+ // int cMin = Integer.MAX_VALUE;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int[] threshold = null;
+
+ switch(block_size) {
+ case 32:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ case 64:
+ threshold = new int[] {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ break;
+ case 128:
+ threshold = new int[] {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ break;
+ case 256:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ break;
+ case 512:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ break;
+ case 1024:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ break;
+ case 2048:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ break;
+ case 4096:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ break;
+ case 8192:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ break;
+ default:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ }
+
+ int cost1 = 0;
+
+ // System.out.println("x:");
+ // for (int i = 0; i < x_length; i++) {
+ // System.out.print(x[i] + " ");
+ // }
+ // System.out.println();
+
+ BitSet[] bitsets = new BitSet[m];
+
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+
+ // for (int i = m - 1; i > 0; i--) {
+ for (int i = 0; i < m; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int current_value = (x[0] >> i) & 1;
+
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ int count = 0;
+
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+
+ for (int j = 1; j < x_length; j++) {
+ // if (count * (1 + (int) Math.ceil(Math.log(x_length))) >= x_length) {
+ // rle_cost_single[i] = x_length + 1;
+ // break;
+ // }
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+
+ bitsets[i].set(j - 1);
+ }
+
+ }
+
+ bitsets[i].set(x_length - 1);
+
+ count++;
+
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+
+ // System.out.println("bpe_cost_single: " + bpe_cost_single[i]);
+ // System.out.println("rle_cost_single: " + rle_cost_single[i]);
+ // System.out.println("de_cost_single: " + de_cost_single[i]);
+
+ if (bpe_cost_single[i] <= rle_cost_single[i] && bpe_cost_single[i] <= de_cost_single[i]) {
+ cost1 += bpe_cost_single[i];
+ } else if (rle_cost_single[i] < bpe_cost_single[i] && rle_cost_single[i] <= de_cost_single[i]) {
+ cost1 += rle_cost_single[i];
+ } else {
+ cost1 += de_cost_single[i];
+ }
+
+ // cost1 += Math.min(bpe_cost_single[i], Math.min(rle_cost_single[i], de_cost_single[i]));
+ }
+
+ int cMin = cost1;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ // int[] beta_list = new int[m - 1];
+ // for (int i = 0; i < m - 1; i++) {
+ // beta_list[i] = i + 2;
+ // }
+
+ // int bw = bitWidth(block_size);
+
+ // int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ // int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ // for (int i = 0; i < l; i++) {
+ // int maxValuePart = 0;
+ // for (int j = 0; j < x_length; j++) {
+ // subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (subcolumnList[i][j] > maxValuePart) {
+ // maxValuePart = subcolumnList[i][j];
+ // }
+ // }
+ // bitWidthListList[i] = bitWidth(maxValuePart);
+ // }
+
+ // for (int i = l - 1; i > 0; i--) {
+ for (int i = 0; i < l; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int currentCost = 0;
+
+ int bpCost = 0;
+
+ // int partMax = 0;
+ // for (int j = 0; j < x_length; j++) {
+ // int partValue = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (partValue > partMax) {
+ // partMax = partValue;
+ // }
+ // }
+ // bpCost = bitWidth(partMax) * x_length;
+
+ // bpCost = bitWidthListList[i] * x_length;
+
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start >= i * beta && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+
+ if (beta_start < i * beta) {
+ beta_start = i * beta;
+ }
+
+ bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
+ // System.out.println("bpCost: " + bpCost);
+
+ currentCost = bpCost;
+
+ int rleCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ // if (true) {
+ // if (rle_cost_single[i * beta] < currentCost) {
+ int rleCost = 0;
+
+ boolean currentBetter = false;
+
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (currentCost > rleCost) {
+ currentCost = rleCost;
+ }
+ }
+
+ // int index = 0;
+
+ // int currentNumber = (x[0] >> (i * beta)) & ((1 << beta) - 1);
+
+ // for (int j = 1; j < x_length; j++) {
+ // int currentNumber_j = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (currentNumber_j != currentNumber) {
+ // index++;
+ // currentNumber = currentNumber_j;
+ // if (beta * index + bitWidth(x_length) * index >= currentCost) {
+ // currentBetter = true;
+ // break;
+ // }
+ // }
+ // }
+
+ // if (!currentBetter) {
+ // index++;
+
+ // rleCost = beta * index + bitWidth(x_length) * index;
+ // if (currentCost > rleCost) {
+ // currentCost = rleCost;
+
+ // encodingTypeTemp[i] = 1;
+ // }
+
+ // // System.out.println("rleCost: " + rleCost);
+ // }
+ }
+
+ int deCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (de_cost_single[j] > deCostMax) {
+ deCostMax = de_cost_single[j];
+ }
+ }
+
+ // 打印threshold[beta - 1]
+ // System.out.println("threshold cardinality: " + threshold[beta - 1]);
+
+ if (deCostMax < currentCost) {
+ // if (true) {
+ // if (de_cost_single[i * beta] < currentCost) {
+ boolean currentBetter = false;
+ Set<Integer> uniqueValues = new HashSet<>();
+
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+
+ if (deCost < currentCost) {
+ currentCost = deCost;
+
+ }
+ // System.out.println("deCost: " + deCost);
+ }
+
+ }
+
+ // System.out.println("cost_min: " + currentCost);
+
+ cost += currentCost;
+
+ // int rleCost = 0;
+
+ // int lowestBitIndex = 0;
+ // int currentLowestBit = subcolumnList[i][0] & 1;
+
+ // for (int j = 1; j < x_length; j++) {
+ // int lowestBit = subcolumnList[i][j] & 1; // 获取当前元素的最低位
+ // if (lowestBit != currentLowestBit) {
+ // lowestBitIndex++;
+ // currentLowestBit = lowestBit;
+ // }
+ // }
+
+ // if (bw * lowestBitIndex + bitWidthListList[i] * lowestBitIndex >= bpCost) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // int index = 0;
+
+ // boolean bpBest = false;
+
+ // int count = 1;
+ // int currentNumber = subcolumnList[i][0];
+ // int currentNumber = (x[0] >> (i * beta)) & ((1 << beta) - 1);
+
+ // for (int j = 1; j < x_length; j++) {
+ // int currentNumber_j = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ // if (currentNumber_j != currentNumber) {
+ // index++;
+ // currentNumber = currentNumber_j;
+ // }
+ // if (bw * index + bitWidth(x_length) * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+
+ // // if (subcolumnList[i][j] != currentNumber) {
+ // // index++;
+ // // currentNumber = subcolumnList[i][j];
+ // // }
+
+ // // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // // bpBest = true;
+ // // break;
+ // // }
+ // }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ // index++;
+
+ // System.out.println("index: " + index);
+
+ // rleCost = bw * index + bitWidth(x_length) * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ // System.out.println("betaBest: " + betaBest);
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ // System.out.println("maxValue: " + maxValue);
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ // System.out.println("All zero list.");
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // System.out.println("encodingType: ");
+ // for (int i = 0; i < l; i++) {
+ // System.out.print(encodingType[i] + " ");
+ // }
+ // System.out.println();
+
+ // encoded_result 预留大小为 (l + 7) * 2 / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ // for (int i = l - 1; i >= 0; i--) {
+ for (int i = 0; i < l; i++) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+// if(currentNumber == 6){
+// System.out.println("currentNumber == 6 && i==0");
+// System.out.println(uniqueValues);
+// }
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ }
+
+ index++;
+
+ rleCost = bitWidth(block_size) * index + beta[0] * index;
+
+
+ // if (encodingType[i] == 2) {
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ // if(cardinality < Math.pow(2,bitWidthList[i]-1)){
+ // test dictionary encoding
+ int dict_bit_width = bitWidth(cardinality);
+ int dicCost =dict_bit_width *list_length + cardinality*(bitWidthList[i] +dict_bit_width);
+ if(dicCost < rleCost && dicCost< bpCost){
+
+ // if dictionary encoding
+ // int dict_bit_width = bitWidth(cardinality) ;
+ encodingType[i] = 2;
+
+// System.out.println(uniqueValues);
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+ // int[] dict_value_list = new int[cardinality];
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+ // dict_value_list[j] = j;
+ }
+// System.out.println(valueToCode);
+// System.out.println(list_length);
+// System.out.println(beta[0]);
+// System.out.println(Arrays.toString(subcolumnList[i]));
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+ // encode_pos = bitPacking(dict_value_list, dict_bit_width, encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+ // }
+
+ if (bpCost <= rleCost) {
+ // if (encodingType[i] == 0) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ // for (int i = l - 1; i >= 0; i--) {
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if(type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ encode_pos =decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ // if (block_index % 16 == 0) {
+ // if (block_index % 32 == 0) {
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ // int[] encodingType = new int[m];
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+
+ // System.out.println("block_index: " + block_index + " beta: " + beta[0]);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneFastTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneFastTest.java
new file mode 100644
index 0000000..b0359e66
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneFastTest.java
@@ -0,0 +1,556 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.BitSet;
+
+
+public class SubcolumnPruneFastTest {
+
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ private static final int[] THRESHOLD_32 =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25,
+ 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47,
+ 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88,
+ 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154,
+ 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270,
+ 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513,
+ 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971,
+ 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593,
+ 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511,
+ 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731,
+ 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757,
+ 4864};
+
+ /** Reusable buffers for pruning (avoids per-block BitSet / array allocation). */
+ static final class PruneWorkspace {
+ BitSet[] bitsets = new BitSet[32];
+ final BitSet merged = new BitSet();
+ int[] bpeCostSingle = new int[32];
+ int[] rleCostSingle = new int[32];
+ int[] deCostSingle = new int[32];
+ int[] encodingTypeTemp = new int[16];
+ PruneWorkspace() {
+ for (int i = 0; i < bitsets.length; i++) {
+ bitsets[i] = new BitSet();
+ }
+ }
+
+ void ensureM(int m) {
+ if (m > bitsets.length) {
+ int newLen = Math.max(m, bitsets.length * 2);
+ bitsets = Arrays.copyOf(bitsets, newLen);
+ for (int i = 0; i < newLen; i++) {
+ if (bitsets[i] == null) {
+ bitsets[i] = new BitSet();
+ }
+ }
+ bpeCostSingle = Arrays.copyOf(bpeCostSingle, newLen);
+ rleCostSingle = Arrays.copyOf(rleCostSingle, newLen);
+ deCostSingle = Arrays.copyOf(deCostSingle, newLen);
+ }
+ int maxL = (m + 1) / 2;
+ if (encodingTypeTemp.length < maxL) {
+ encodingTypeTemp = new int[maxL];
+ }
+ }
+
+ void clearBitsets(int m) {
+ for (int i = 0; i < m; i++) {
+ bitsets[i].clear();
+ }
+ }
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return THRESHOLD_32;
+ }
+ }
+
+ /**
+ * Same pruning decisions as {@link SubcolumnPruneTest#Subcolumn}; faster via reused
+ * BitSets and int-bitmask dictionary cardinality (beta ≤ 4).
+ */
+ public static int subcolumnPrune(
+ PruneWorkspace ws,
+ int[] x,
+ int xLength,
+ int m,
+ int blockSize,
+ int[] encodingType) {
+ if (m == 0) {
+ return 1;
+ }
+
+ ws.ensureM(m);
+ ws.clearBitsets(m);
+
+ int betaBest = 1;
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = 32 - Integer.numberOfLeadingZeros(xLength);
+ int cost1 = 0;
+
+ int[] bpe = ws.bpeCostSingle;
+ int[] rle = ws.rleCostSingle;
+ int[] de = ws.deCostSingle;
+ BitSet[] bitsets = ws.bitsets;
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnIj = (x[j] >> i) & 1;
+ if (subcolumnIj == 1) {
+ hasOne = true;
+ }
+ if (subcolumnIj != currentValue) {
+ runCount++;
+ currentValue = subcolumnIj;
+ changed = true;
+ bitsets[i].set(j - 1);
+ }
+ }
+
+ bitsets[i].set(xLength - 1);
+ bpe[i] = hasOne ? xLength : 0;
+ rle[i] = runCount * (1 + lengthBitWidth);
+ de[i] = changed ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpe[i] <= rle[i] && bpe[i] <= de[i]) {
+ encodingType[i] = 0;
+ cost1 += bpe[i];
+ } else if (rle[i] < bpe[i] && rle[i] <= de[i]) {
+ encodingType[i] = 1;
+ cost1 += rle[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += de[i];
+ }
+ }
+
+ int cMin = cost1;
+ BitSet merged = ws.merged;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = ws.encodingTypeTemp;
+ Arrays.fill(encodingTypeTemp, 0, l, 0);
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+
+ int betaStart = Math.min(m - 1, groupEnd - 1);
+ while (betaStart >= groupStart && bpe[betaStart] == 0) {
+ betaStart--;
+ }
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpe[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rle[j] > rleCostMax) {
+ rleCostMax = rle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ merged.clear();
+ boolean currentBetter = false;
+ for (int j = groupStart; j < groupEnd; j++) {
+ merged.or(bitsets[j]);
+ if (merged.cardinality() >= currentCost) {
+ currentBetter = true;
+ break;
+ }
+ }
+ if (!currentBetter) {
+ int rleCost = merged.cardinality() * (beta + lengthBitWidth);
+ if (currentCost > rleCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (de[j] > deCostMax) {
+ deCostMax = de[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int dictLimit = threshold[beta - 1];
+ int seenMask = 0;
+ int uniqueSize = 0;
+ for (int j = 0; j < xLength; j++) {
+ int currentNumber = (x[j] >> groupStart) & mask;
+ int bit = 1 << currentNumber;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ uniqueSize++;
+ if (uniqueSize >= dictLimit) {
+ uniqueSize = dictLimit;
+ break;
+ }
+ }
+ }
+ if (uniqueSize < dictLimit) {
+ int deCost =
+ xLength * (32 - Integer.numberOfLeadingZeros(uniqueSize))
+ + uniqueSize * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int BlockEncoder(
+ PruneWorkspace ws,
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta) {
+ int[] minDelta = new int[1];
+ int[] dataDelta =
+ SubcolumnPruneNewTest.getAbsDeltaTsBlock(
+ data, blockIndex, blockSize, remainder, minDelta);
+
+ SubcolumnPruneNewTest.int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ int m = SubcolumnPruneNewTest.bitWidth(maxValue);
+ int[] encodingType = new int[m];
+ beta[0] = subcolumnPrune(ws, dataDelta, remainder, m, blockSize, encodingType);
+
+ return SubcolumnPruneNewTest.SubcolumnEncoder(
+ dataDelta, encodePos, encodedResult, beta, blockSize, encodingType);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ SubcolumnPruneNewTest.int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ SubcolumnPruneNewTest.int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+ PruneWorkspace ws = new PruneWorkspace();
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockEncoder(
+ ws, data, i, blockSize, blockSize, encodePos, encodedResult, beta);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ SubcolumnPruneNewTest.int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockEncoder(
+ ws, data, numBlocks, blockSize, remainder, encodePos, encodedResult,
+ beta);
+ }
+
+ return encodePos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ /** Byte-identical to {@link SubcolumnPruneTest#Encoder} on synthetic blocks. */
+ @Test
+ public void testEncodedBytesMatchPruneTest() {
+ int[] blockSizes = {32, 64, 128, 256, 512, 1024};
+ int[][] patterns = {
+ {0, 0, 0, 0},
+ {1, 2, 3, 4, 5, 6, 7, 8},
+ {100, 100, 101, 102, 103, 100, 100},
+ {7, 7, 7, 8, 8, 9, 9, 9, 7, 7},
+ };
+
+ for (int blockSize : blockSizes) {
+ for (int[] pattern : patterns) {
+ int[] data = new int[blockSize];
+ for (int i = 0; i < blockSize; i++) {
+ data[i] = pattern[i % pattern.length] + (i % 17);
+ }
+ byte[] ref = new byte[data.length * 13];
+ byte[] fast = new byte[data.length * 13];
+ int refLen = SubcolumnPruneTest.Encoder(data, blockSize, ref);
+ int fastLen = Encoder(data, blockSize, fast);
+ Assert.assertEquals(
+ "length mismatch blockSize=" + blockSize,
+ refLen,
+ fastLen);
+ Assert.assertArrayEquals(
+ "bytes mismatch blockSize=" + blockSize,
+ Arrays.copyOf(ref, refLen),
+ Arrays.copyOf(fast, fastLen));
+ }
+ }
+ }
+
+ /** Spot-check against PruneNew (should match if New pruning equals Prune). */
+ @Test
+ public void testEncodedBytesMatchPruneNewTest() {
+ int blockSize = 512;
+ int[] data = new int[blockSize];
+ for (int i = 0; i < blockSize; i++) {
+ data[i] = (i * 37 + 11) % 1000;
+ }
+ byte[] ref = new byte[data.length * 13];
+ byte[] fast = new byte[data.length * 13];
+ int refLen = SubcolumnPruneNewTest.Encoder(data, blockSize, ref);
+ int fastLen = Encoder(data, blockSize, fast);
+ if (refLen != fastLen || !Arrays.equals(Arrays.copyOf(ref, refLen), Arrays.copyOf(fast, fastLen))) {
+ System.out.println(
+ "Fast vs PruneNew: bytes differ (expected if RLE merge semantics differ).");
+ }
+ }
+
+
+ @Test
+ public void testBlockSizeBenchmark() throws IOException {
+ // String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parentDir = "D:/github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/compression_vs_block_noprune_legacy/";
+
+ File outputDir = new File(outputParentDir);
+ if (!outputDir.exists() && !outputDir.mkdirs()) {
+ throw new IOException("Cannot create " + outputParentDir);
+ }
+
+ int[] blockSizeList = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+ int repeatTime = 100;
+
+ String[] datasets = {
+ "Bird-migration",
+ "Bitcoin-price",
+ "City-temp",
+ "Dewpoint-temp",
+ "EPM-Education",
+ "Gov10",
+ "IR-bio-temp",
+ "PM10-dust",
+ "Stocks-DE",
+ "Stocks-UK",
+ "Stocks-USA",
+ "Wind-Speed",
+ "Wine-Tasting"
+ };
+
+ PruneWorkspace ws = new PruneWorkspace();
+ final int firstBlockSize = blockSizeList[0];
+
+ for (int blockSize : blockSizeList) {
+ final boolean warmupBeforeTiming = blockSize == firstBlockSize;
+ String outputPath = outputParentDir + "subcolumn_block_" + blockSize + ".csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ for (String datasetName : datasets) {
+ File file = new File(inputParentDir + datasetName + ".csv");
+ if (!file.exists()) {
+ System.out.println("Skip missing: " + file);
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ java.util.ArrayList<Double> data1 = new java.util.ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, getDecimalPrecision(fStr));
+ data1.add(Double.valueOf(fStr));
+ }
+ loader.close();
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ long maxMul = (long) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encoded = new byte[data2Arr.length * 13];
+ byte[] encodedRef = new byte[data2Arr.length * 13];
+
+ int length =
+ SubcolumnPruneTest.Encoder(data2Arr, blockSize, encodedRef);
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < repeatTime; r++) {
+ Encoder(data2Arr, blockSize, encoded);
+ }
+ }
+
+ long s = System.nanoTime();
+ int fastLen = 0;
+ for (int r = 0; r < repeatTime; r++) {
+ fastLen = Encoder(data2Arr, blockSize, encoded);
+ }
+ long encodeTime = (System.nanoTime() - s) / repeatTime;
+
+ Assert.assertEquals(
+ "compressed size must match PruneTest: "
+ + datasetName
+ + " block="
+ + blockSize,
+ length,
+ fastLen);
+ Assert.assertArrayEquals(
+ Arrays.copyOf(encodedRef, length),
+ Arrays.copyOf(encoded, fastLen));
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < repeatTime; r++) {
+ SubcolumnPruneTest.Decoder(encoded);
+ }
+ }
+
+ s = System.nanoTime();
+ for (int r = 0; r < repeatTime; r++) {
+ SubcolumnPruneTest.Decoder(encoded);
+ }
+ long decodeTime = (System.nanoTime() - s) / repeatTime;
+
+ double ratio = length / (double) (data1.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(length),
+ String.valueOf(ratio)
+ });
+ System.out.println(
+ datasetName
+ + " block="
+ + blockSize
+ + " encode_ns="
+ + encodeTime
+ + " ratio="
+ + ratio);
+ }
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBenchmarkTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBenchmarkTest.java
new file mode 100644
index 0000000..2584f5c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBenchmarkTest.java
@@ -0,0 +1,120 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class SubcolumnPruneNewBenchmarkTest {
+
+ private static int[] syntheticData(int size, int seed) {
+ int[] data = new int[size];
+ int state = seed;
+ for (int i = 0; i < size; i++) {
+ state = state * 1103515245 + 12345;
+ data[i] = (state >>> 16) & 0x7FFF;
+ if (i % 17 == 0) {
+ data[i] = data[Math.max(0, i - 1)];
+ }
+ }
+ return data;
+ }
+
+ @Test
+ public void benchmarkSubcolumnPruneNew() {
+ int blockSize = 512;
+ int repeatTime = 200;
+ int[] data = syntheticData(512 * 200, 99);
+ byte[] encoded = new byte[data.length * 8];
+
+ int encodedLen = SubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+
+ long encodeTotal = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ encodedLen = SubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+ encodeTotal += System.nanoTime() - start;
+ }
+
+ long decodeTotal = 0;
+ byte[] encodedCopy = Arrays.copyOf(encoded, encodedLen);
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ SubcolumnPruneNewTest.Decoder(encodedCopy);
+ decodeTotal += System.nanoTime() - start;
+ }
+
+ double avgEncodeMs = encodeTotal / (double) repeatTime / 1_000_000.0;
+ double avgDecodeMs = decodeTotal / (double) repeatTime / 1_000_000.0;
+ System.out.println("SubcolumnPruneNew benchmark (points=" + data.length
+ + ", blockSize=" + blockSize + ", repeats=" + repeatTime + ")");
+ System.out.println("avg encode ms: " + avgEncodeMs);
+ System.out.println("avg decode ms: " + avgDecodeMs);
+ System.out.println("compressed bytes: " + encodedLen);
+ }
+
+ @Test
+ public void benchmarkSprintzSubcolumn() {
+ int blockSize = 512;
+ int repeatTime = 200;
+ int[] data = syntheticData(512 * 200, 99);
+ byte[] encoded = new byte[data.length * 8];
+
+ int encodedLen = SPRINTZSubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+
+ long encodeTotal = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ encodedLen = SPRINTZSubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+ encodeTotal += System.nanoTime() - start;
+ }
+
+ long decodeTotal = 0;
+ byte[] encodedCopy = Arrays.copyOf(encoded, encodedLen);
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ SPRINTZSubcolumnPruneNewTest.Decoder(encodedCopy);
+ decodeTotal += System.nanoTime() - start;
+ }
+
+ double avgEncodeMs = encodeTotal / (double) repeatTime / 1_000_000.0;
+ double avgDecodeMs = decodeTotal / (double) repeatTime / 1_000_000.0;
+ System.out.println("SPRINTZ+Subcolumn benchmark (points=" + data.length
+ + ", blockSize=" + blockSize + ", repeats=" + repeatTime + ")");
+ System.out.println("avg encode ms: " + avgEncodeMs);
+ System.out.println("avg decode ms: " + avgDecodeMs);
+ System.out.println("compressed bytes: " + encodedLen);
+ }
+
+ @Test
+ public void benchmarkTs2diffSubcolumn() {
+ int blockSize = 512;
+ int repeatTime = 200;
+ int[] data = syntheticData(512 * 200, 99);
+ byte[] encoded = new byte[data.length * 8];
+
+ int encodedLen = TSDIFFSubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+
+ long encodeTotal = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ encodedLen = TSDIFFSubcolumnPruneNewTest.Encoder(data, blockSize, encoded);
+ encodeTotal += System.nanoTime() - start;
+ }
+
+ long decodeTotal = 0;
+ byte[] encodedCopy = Arrays.copyOf(encoded, encodedLen);
+ for (int i = 0; i < repeatTime; i++) {
+ long start = System.nanoTime();
+ TSDIFFSubcolumnPruneNewTest.Decoder(encodedCopy);
+ decodeTotal += System.nanoTime() - start;
+ }
+
+ double avgEncodeMs = encodeTotal / (double) repeatTime / 1_000_000.0;
+ double avgDecodeMs = decodeTotal / (double) repeatTime / 1_000_000.0;
+ System.out.println("TS2DIFF+Subcolumn benchmark (points=" + data.length
+ + ", blockSize=" + blockSize + ", repeats=" + repeatTime + ")");
+ System.out.println("avg encode ms: " + avgEncodeMs);
+ System.out.println("avg decode ms: " + avgDecodeMs);
+ System.out.println("compressed bytes: " + encodedLen);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBetaTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBetaTest.java
new file mode 100644
index 0000000..34df593
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBetaTest.java
@@ -0,0 +1,1027 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SubcolumnPruneNewBetaTest {
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(int[] values, int length, int shiftAmount,
+ int mask, int limit) {
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize, int[] encodingType) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnValue = (x[j] >> i) & 1;
+ if (subcolumnValue == 1) {
+ hasOne = true;
+ }
+ if (subcolumnValue != currentValue) {
+ runCount++;
+ currentValue = subcolumnValue;
+ changed = true;
+ }
+ }
+
+ bpeCostSingle[i] = hasOne ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = changed ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 0;
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 1;
+ cost1 += rleCostSingle[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = new int[l];
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ int runCount = countGroupedRuns(x, xLength, groupStart, mask);
+ int rleCost = runCount * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount = countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, threshold[beta - 1]);
+ if (distinctCount < threshold[beta - 1]) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ return betaBest;
+ }
+
+ /**
+ * Pick per-group encoding types (BPE/RLE/DE) for a fixed beta, using the same prune rules as
+ * {@link #Subcolumn} but without searching {@link #BETA_LIST}.
+ */
+ public static void fillEncodingTypeForFixedBeta(
+ int[] x,
+ int xLength,
+ int m,
+ int blockSize,
+ int beta,
+ int[] encodingType) {
+ if (m == 0 || beta > m) {
+ return;
+ }
+
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnValue = (x[j] >> i) & 1;
+ if (subcolumnValue == 1) {
+ hasOne = true;
+ }
+ if (subcolumnValue != currentValue) {
+ runCount++;
+ currentValue = subcolumnValue;
+ changed = true;
+ }
+ }
+
+ bpeCostSingle[i] = hasOne ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = changed ? xLength * 2 + 2 : xLength + 2;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+ encodingType[i] = 0;
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ int runCount = countGroupedRuns(x, xLength, groupStart, mask);
+ int rleCost = runCount * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingType[i] = 1;
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount =
+ countDistinctValuesUntilLimit(x, xLength, groupStart, mask, threshold[beta - 1]);
+ if (distinctCount < threshold[beta - 1]) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ encodingType[i] = 2;
+ }
+ }
+ }
+ }
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType) {
+ int listLength = list.length;
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = new int[l];
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+ boolean[] seenValues = new boolean[mask + 1];
+ int[] dictKeyList = new int[mask + 1];
+ int[] codeMap = new int[mask + 1];
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+
+ if (encodingType[i] == 2) {
+ Arrays.fill(seenValues, false);
+ int cardinality = 0;
+ for (int j = 0; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (!seenValues[current]) {
+ seenValues[current] = true;
+ cardinality++;
+ }
+ }
+
+ int dictBitWidth = bitWidth(cardinality);
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if (seenValues[value]) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+ encodePos = bitPacking(subcolumnBuffer, bitWidthList[i], encodePos, encodedResult,
+ listLength);
+ } else {
+ int previous = subcolumnBuffer[0];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(runLength, bw, encodePos, encodedResult, runCount);
+ encodePos = bitPacking(rleValues, bitWidthList[i], encodePos, encodedResult,
+ runCount);
+ }
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ int listLength = list.length;
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+
+ if (type == 0) {
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ listLength, subcolumnBuffer);
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j];
+ while (currentIndex < endPos) {
+ subcolumnBuffer[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = dictKeyList[subcolumnBuffer[j]];
+ }
+ }
+
+ int shiftAmount = i * beta;
+ for (int j = 0; j < listLength; j++) {
+ list[j] |= subcolumnBuffer[j] << shiftAmount;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ tsBlockDelta[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return tsBlockDelta;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ int[] minDelta = new int[1];
+ int[] dataDelta = getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] encodingType = new int[l];
+ fillEncodingTypeForFixedBeta(dataDelta, remainder, m, blockSize, betaValue, encodingType);
+
+ return SubcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize,
+ encodingType);
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int[] blockData = new int[remainder];
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, blockData, blockSize);
+
+ int base = blockIndex * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = blockData[i] + minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, 2);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, int betaValue) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {betaValue};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult, beta);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockEncoder(data, numBlocks, blockSize, remainder, encodePos, encodedResult, beta);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testCompressionVsBeta() throws IOException {
+ // String parentDir = "path/to/your/directory/";
+ String parentDir = "D:/github/xjz17/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+
+ String outputParentDir = parentDir + "result/compression_vs_beta_prune/";
+
+ File outputDir = new File(outputParentDir);
+ if (!outputDir.exists()) {
+ outputDir.mkdirs();
+ }
+
+ int[] betaList = {
+ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
+ 25, 26, 27, 28, 29, 30, 31
+ };
+
+ int blockSize = 1024;
+ int repeatTime = 200;
+
+ List<String> datasetList = new ArrayList<>();
+ datasetList.add("Arade4");
+ datasetList.add("Bird-migration");
+ datasetList.add("Bitcoin-price");
+ datasetList.add("Census-Population");
+ datasetList.add("City-temp");
+ datasetList.add("Dewpoint-temp");
+ datasetList.add("EPM-Education");
+ datasetList.add("Gov10");
+ datasetList.add("MeteoNet-Weather");
+ // datasetList.add("POI-lat");
+ datasetList.add("IR-bio-temp");
+ datasetList.add("PM10-dust");
+ datasetList.add("Stocks-DE");
+ datasetList.add("Stocks-UK");
+ datasetList.add("Stocks-USA");
+ datasetList.add("Wind-Speed");
+ datasetList.add("Wine-Tasting");
+
+ for (int beta : betaList) {
+ String outputPath = outputParentDir + "subcolumn_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ for (String datasetName : datasetList) {
+ String filePath = inputParentDir + datasetName + ".csv";
+ File file = new File(filePath);
+
+ if (!file.exists()) {
+ System.out.println("File not found: " + filePath);
+ continue;
+ }
+
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data1.add(Double.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ long maxMul = (long) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[data2Arr.length * 13];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressedSize = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2Arr, blockSize, encodedResult, beta);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressedSize += length;
+
+ ratio += compressedSize / (double) (data1.size() * Long.BYTES);
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeNoPruneTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeNoPruneTest.java
new file mode 100644
index 0000000..b73fd0d
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeNoPruneTest.java
@@ -0,0 +1,756 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+
+public class SubcolumnPruneNewBlockSizeNoPruneTest {
+
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ private static final int[] THRESHOLD_32 =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25,
+ 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47,
+ 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88,
+ 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154,
+ 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270,
+ 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513,
+ 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971,
+ 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593,
+ 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511,
+ 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731,
+ 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757,
+ 4864};
+
+ private static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ private static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ private static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ private static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ private static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ private static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ private static int bitPacking(int[] numbers, int bitWidth, int encodePos, byte[] encodedResult,
+ int numValues) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ private static int bitPackingFromList(
+ ArrayList<Integer> numbers, int bitWidth, int encodePos, byte[] encodedResult) {
+ int numValues = numbers.size();
+ int[] packed = new int[numValues];
+ for (int i = 0; i < numValues; i++) {
+ packed[i] = numbers.get(i);
+ }
+ return bitPacking(packed, bitWidth, encodePos, encodedResult, numValues);
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return THRESHOLD_32;
+ }
+ }
+
+ private static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ tsBlockDelta[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return tsBlockDelta;
+ }
+
+ private static int countTrue(boolean[] flags) {
+ int count = 0;
+ for (boolean flag : flags) {
+ if (flag) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private static void orMerge(boolean[] merged, boolean[] column) {
+ for (int p = 0; p < merged.length; p++) {
+ if (column[p]) {
+ merged[p] = true;
+ }
+ }
+ }
+
+ /** Same beta decisions as {@link SubcolumnPruneTest#Subcolumn}; uses boolean[] / ArrayList. */
+ private static int selectBetaAndEncodingTypes(int[] x, int xLength, int m, int blockSize,
+ int[] encodingType) {
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+ boolean[][] bitsets = new boolean[m][xLength];
+
+ int cost1 = 0;
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ if (currentValue == 1) {
+ bpeCostSingle[i] = xLength;
+ }
+
+ int count = 0;
+ deCostSingle[i] = xLength + 2;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnIj = (x[j] >> i) & 1;
+ if (subcolumnIj == 1) {
+ bpeCostSingle[i] = xLength;
+ }
+ if (subcolumnIj != currentValue) {
+ count++;
+ currentValue = subcolumnIj;
+ deCostSingle[i] = xLength * 2 + 2;
+ bitsets[i][j - 1] = true;
+ }
+ }
+
+ bitsets[i][xLength - 1] = true;
+ count++;
+ rleCostSingle[i] = count * (1 + lengthBitWidth);
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 0;
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 1;
+ cost1 += rleCostSingle[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = new int[l];
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+
+ int betaStart = Math.min(m - 1, groupEnd - 1);
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ boolean[] mergedFlags = new boolean[xLength];
+ boolean currentBetter = false;
+ for (int j = groupStart; j < groupEnd; j++) {
+ orMerge(mergedFlags, bitsets[j]);
+ if (countTrue(mergedFlags) >= currentCost) {
+ currentBetter = true;
+ break;
+ }
+ }
+ if (!currentBetter) {
+ int rleCost = countTrue(mergedFlags) * (beta + lengthBitWidth);
+ if (currentCost > rleCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ ArrayList<Integer> uniqueValues = new ArrayList<>();
+ boolean currentBetter = false;
+ for (int j = 0; j < xLength; j++) {
+ int currentNumber = (x[j] >> groupStart) & mask;
+ if (!uniqueValues.contains(currentNumber)) {
+ uniqueValues.add(currentNumber);
+ }
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ currentBetter = true;
+ break;
+ }
+ }
+ if (!currentBetter) {
+ int deCost =
+ xLength * bitWidth(uniqueValues.size())
+ + uniqueValues.size() * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ return betaBest;
+ }
+
+ private static int subcolumnEncoder(int[] list, int encodePos, byte[] encodedResult, int[] beta,
+ int blockSize, int[] encodingType) {
+ int listLength = list.length;
+
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ ArrayList<Integer> bitWidthList = new ArrayList<>(l);
+ ArrayList<ArrayList<Integer>> subcolumnList = new ArrayList<>(l);
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+
+ for (int i = 0; i < l; i++) {
+ ArrayList<Integer> groupValues = new ArrayList<>(listLength);
+ int maxValuePart = 0;
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ int part = (list[j] >> shiftAmount) & mask;
+ groupValues.add(part);
+ if (part > maxValuePart) {
+ maxValuePart = part;
+ }
+ }
+ subcolumnList.add(groupValues);
+ bitWidthList.add(bitWidth(maxValuePart));
+ }
+
+ encodePos = bitPackingFromList(bitWidthList, 8, encodePos, encodedResult);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+ ArrayList<Integer> subcolumnBuffer = subcolumnList.get(i);
+
+ if (encodingType[i] == 2) {
+ boolean[] seenValues = new boolean[mask + 1];
+ for (int j = 0; j < listLength; j++) {
+ seenValues[subcolumnBuffer.get(j)] = true;
+ }
+ ArrayList<Integer> dictKeys = new ArrayList<>();
+ for (int value = 0; value <= mask; value++) {
+ if (seenValues[value]) {
+ dictKeys.add(value);
+ }
+ }
+ int cardinality = dictKeys.size();
+ int dictBitWidth = bitWidth(cardinality);
+
+ for (int j = 0; j < listLength; j++) {
+ int current = subcolumnBuffer.get(j);
+ int code = dictKeys.indexOf(current);
+ subcolumnBuffer.set(j, code);
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos =
+ bitPackingFromList(
+ dictKeys, bitWidthList.get(i), encodePos, encodedResult);
+ encodePos =
+ bitPackingFromList(subcolumnBuffer, dictBitWidth, encodePos, encodedResult);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+ encodePos =
+ bitPackingFromList(
+ subcolumnBuffer, bitWidthList.get(i), encodePos, encodedResult);
+ } else {
+ ArrayList<Integer> runLength = new ArrayList<>();
+ ArrayList<Integer> rleValues = new ArrayList<>();
+ int previous = subcolumnBuffer.get(0);
+
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer.get(j);
+ if (current != previous) {
+ runLength.add(j);
+ rleValues.add(previous);
+ previous = current;
+ }
+ }
+
+ runLength.add(listLength);
+ rleValues.add(previous);
+ int runCount = runLength.size();
+
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPackingFromList(runLength, bw, encodePos, encodedResult);
+ encodePos =
+ bitPackingFromList(
+ rleValues, bitWidthList.get(i), encodePos, encodedResult);
+ }
+ }
+
+ ArrayList<Integer> encodingTypeList = new ArrayList<>(l);
+ for (int i = 0; i < l; i++) {
+ encodingTypeList.add(encodingType[i]);
+ }
+ bitPackingFromList(encodingTypeList, 2, preTypePos, encodedResult);
+ return encodePos;
+ }
+
+ private static int blockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ int[] minDelta = new int[1];
+ int[] dataDelta = getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ int[] encodingType = new int[m];
+ beta[0] = selectBetaAndEncodingTypes(dataDelta, remainder, m, blockSize, encodingType);
+
+ return subcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize, encodingType);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = blockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult, beta);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ blockEncoder(
+ data, numBlocks, blockSize, remainder, encodePos, encodedResult, beta);
+ }
+
+ return encodePos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ @Test
+ public void testEncodedBytesMatchPruneFastTest() {
+ int[] blockSizes = {32, 64, 128, 256, 512, 1024};
+ int[][] patterns = {
+ {0, 0, 0, 0},
+ {1, 2, 3, 4, 5, 6, 7, 8},
+ {100, 100, 101, 102, 103, 100, 100},
+ {7, 7, 7, 8, 8, 9, 9, 9, 7, 7},
+ };
+
+ for (int blockSize : blockSizes) {
+ for (int[] pattern : patterns) {
+ int[] data = new int[blockSize];
+ for (int i = 0; i < blockSize; i++) {
+ data[i] = pattern[i % pattern.length] + (i % 17);
+ }
+ byte[] ref = new byte[data.length * 13];
+ byte[] plain = new byte[data.length * 13];
+ int refLen = SubcolumnPruneFastTest.Encoder(data, blockSize, ref);
+ int plainLen = Encoder(data, blockSize, plain);
+ Assert.assertEquals("length mismatch blockSize=" + blockSize, refLen, plainLen);
+ Assert.assertArrayEquals(
+ "bytes mismatch blockSize=" + blockSize,
+ Arrays.copyOf(ref, refLen),
+ Arrays.copyOf(plain, plainLen));
+ }
+ }
+ }
+
+ @Test
+ public void testBlockSizeBenchmark() throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/compression_vs_block_noprune/";
+
+ File outputDir = new File(outputParentDir);
+ if (!outputDir.exists() && !outputDir.mkdirs()) {
+ throw new IOException("Cannot create " + outputParentDir);
+ }
+
+ int[] blockSizeList = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+ int repeatTime = 400;
+ int decodeRepeatTime = 400;
+
+ String[] datasets = {
+ "Bird-migration",
+ "Bitcoin-price",
+ "City-temp",
+ "Dewpoint-temp",
+ "EPM-Education",
+ "Gov10",
+ "IR-bio-temp",
+ "PM10-dust",
+ "Stocks-DE",
+ "Stocks-UK",
+ "Stocks-USA",
+ "Wind-Speed",
+ "Wine-Tasting"
+ };
+
+ final int firstBlockSize = blockSizeList[0];
+
+ for (int blockSize : blockSizeList) {
+ final boolean warmupBeforeTiming = blockSize == firstBlockSize;
+ String outputPath = outputParentDir + "subcolumn_block_" + blockSize + ".csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ for (String datasetName : datasets) {
+ File file = new File(inputParentDir + datasetName + ".csv");
+ if (!file.exists()) {
+ System.out.println("Skip missing: " + file);
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ java.util.ArrayList<Double> data1 = new java.util.ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(maxDecimal, getDecimalPrecision(fStr));
+ data1.add(Double.valueOf(fStr));
+ }
+ loader.close();
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ long maxMul = (long) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encoded = new byte[data2Arr.length * 13];
+ byte[] encodedRef = new byte[data2Arr.length * 13];
+
+ int length =
+ SubcolumnPruneFastTest.Encoder(data2Arr, blockSize, encodedRef);
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < repeatTime; r++) {
+ Encoder(data2Arr, blockSize, encoded);
+ }
+ }
+
+ long s = System.nanoTime();
+ int encodedLen = 0;
+ for (int r = 0; r < repeatTime; r++) {
+ encodedLen = Encoder(data2Arr, blockSize, encoded);
+ }
+ long encodeTime = (System.nanoTime() - s) / repeatTime;
+
+ Assert.assertEquals(
+ "compressed size must match PruneFastTest: "
+ + datasetName
+ + " block="
+ + blockSize,
+ length,
+ encodedLen);
+ Assert.assertArrayEquals(
+ Arrays.copyOf(encodedRef, length),
+ Arrays.copyOf(encoded, encodedLen));
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < decodeRepeatTime; r++) {
+ SubcolumnPruneNewTest.Decoder(encoded);
+ }
+ }
+
+ s = System.nanoTime();
+ for (int r = 0; r < decodeRepeatTime; r++) {
+ SubcolumnPruneNewTest.Decoder(encoded);
+ }
+ long decodeTime = (System.nanoTime() - s) / decodeRepeatTime;
+
+ double ratio = length / (double) (data1.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(length),
+ String.valueOf(ratio)
+ });
+ System.out.println(
+ datasetName
+ + " block="
+ + blockSize
+ + " encode_ns="
+ + encodeTime
+ + " ratio="
+ + ratio);
+ }
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeTest.java
new file mode 100644
index 0000000..6a78254
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewBlockSizeTest.java
@@ -0,0 +1,149 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class SubcolumnPruneNewBlockSizeTest {
+
+ @Test
+ public void testBlockSizeBenchmark() throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/compression_vs_block_prune_fast/";
+
+ File outputDir = new File(outputParentDir);
+ if (!outputDir.exists() && !outputDir.mkdirs()) {
+ throw new IOException("Cannot create " + outputParentDir);
+ }
+
+ int[] blockSizeList = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+ int repeatTime = 400;
+ int decodeRepeatTime = 400;
+
+ String[] datasets = {
+ "Bird-migration",
+ "Bitcoin-price",
+ "City-temp",
+ "Dewpoint-temp",
+ "EPM-Education",
+ "Gov10",
+ "IR-bio-temp",
+ "PM10-dust",
+ "Stocks-DE",
+ "Stocks-UK",
+ "Stocks-USA",
+ "Wind-Speed",
+ "Wine-Tasting"
+ };
+
+ final int firstBlockSize = blockSizeList[0];
+
+ for (int blockSize : blockSizeList) {
+ final boolean warmupBeforeTiming = blockSize == firstBlockSize;
+ String outputPath = outputParentDir + "subcolumn_block_" + blockSize + ".csv";
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ for (String datasetName : datasets) {
+ File file = new File(inputParentDir + datasetName + ".csv");
+ if (!file.exists()) {
+ System.out.println("Skip missing: " + file);
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ maxDecimal = Math.max(
+ maxDecimal, SubcolumnPruneNewTest.getDecimalPrecision(fStr));
+ data1.add(Float.valueOf(fStr));
+ }
+ loader.close();
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encoded = new byte[Math.max(16, data2Arr.length * 8)];
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < repeatTime; r++) {
+ SubcolumnPruneNewTest.Encoder(data2Arr, blockSize, encoded);
+ }
+ }
+
+ long s = System.nanoTime();
+ int encodedLen = 0;
+ for (int r = 0; r < repeatTime; r++) {
+ encodedLen = SubcolumnPruneNewTest.Encoder(data2Arr, blockSize, encoded);
+ }
+ long encodeTime = (System.nanoTime() - s) / repeatTime;
+
+ if (warmupBeforeTiming) {
+ for (int r = 0; r < decodeRepeatTime; r++) {
+ SubcolumnPruneNewTest.Decoder(encoded);
+ }
+ }
+
+ s = System.nanoTime();
+ for (int r = 0; r < decodeRepeatTime; r++) {
+ SubcolumnPruneNewTest.Decoder(encoded);
+ }
+ long decodeTime = (System.nanoTime() - s) / decodeRepeatTime;
+
+ double ratio = encodedLen / (double) (data1.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(encodedLen),
+ String.valueOf(ratio)
+ });
+ System.out.println(
+ datasetName
+ + " block="
+ + blockSize
+ + " encode_ns="
+ + encodeTime
+ + " ratio="
+ + ratio);
+ }
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewEncodingTypeRatioTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewEncodingTypeRatioTest.java
new file mode 100644
index 0000000..84a2f81
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewEncodingTypeRatioTest.java
@@ -0,0 +1,1142 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+public class SubcolumnPruneNewEncodingTypeRatioTest {
+
+ private static class EncodingTypeStats {
+ private int totalBlockCount;
+ private int totalSubcolumnCount;
+ private final int[] encodingTypeCounts = new int[3];
+
+ private void setTotalBlockCount(int totalBlockCount) {
+ this.totalBlockCount = totalBlockCount;
+ }
+
+ /**
+ * Omits Bit Packing rows where the grouped subcolumn needs the full {@code beta} bits
+ * (effective max width equals {@code beta}: no redundant leading-zero MSBs to strip). Those
+ * are excluded from Bit Packing counts and from the subcolumn denominator; BP rows with
+ * {@code bitWidthList[i] < beta} still count.
+ */
+ private void recordEncodingType(
+ int[] encodingType, int length, int beta, int[] bitWidthList) {
+ if (length <= 0) {
+ return;
+ }
+
+ for (int i = 0; i < length; i++) {
+ int currentType = encodingType[i];
+ if (currentType == 0
+ && bitWidthList != null
+ && i < bitWidthList.length
+ && bitWidthList[i] == beta) {
+ continue;
+ }
+ totalSubcolumnCount++;
+ if (currentType >= 0 && currentType < encodingTypeCounts.length) {
+ encodingTypeCounts[currentType]++;
+ }
+ }
+ }
+
+ private int getTotalBlockCount() {
+ return totalBlockCount;
+ }
+
+ private int getTotalSubcolumnCount() {
+ return totalSubcolumnCount;
+ }
+
+ private int getBitPackingSubcolumnCount() {
+ return encodingTypeCounts[0];
+ }
+
+ private int getRleSubcolumnCount() {
+ return encodingTypeCounts[1];
+ }
+
+ private int getDictionarySubcolumnCount() {
+ return encodingTypeCounts[2];
+ }
+
+ private double getBitPackingRatio() {
+ return getEncodingTypeRatio(0);
+ }
+
+ private double getRleRatio() {
+ return getEncodingTypeRatio(1);
+ }
+
+ private double getDictionaryRatio() {
+ return getEncodingTypeRatio(2);
+ }
+
+ private double getEncodingTypeRatio(int type) {
+ if (totalSubcolumnCount == 0) {
+ return 0;
+ }
+ return encodingTypeCounts[type] / (double) totalSubcolumnCount;
+ }
+ }
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ private static final int[] BETA_LIST = {2, 3, 4};
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ /**
+ * Per grouped-subcolumn max value bit width; must match {@link #SubcolumnEncoder} {@code
+ * bitWidthList} computation for statistics filtering.
+ */
+ private static int[] computeGroupedMaxBitWidths(
+ int[] dataDelta, int remainder, int m, int betaValue) {
+ if (m <= 0 || betaValue <= 0) {
+ return new int[0];
+ }
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = new int[l];
+ int mask = (1 << betaValue) - 1;
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ int maxValuePart = 0;
+ for (int j = 0; j < remainder; j++) {
+ int current = (dataDelta[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+ return bitWidthList;
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(int[] values, int length, int shiftAmount,
+ int mask, int limit) {
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize, int[] encodingType) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ // int betaBest = 2;
+
+ int[] bpeCostSingle = new int[m];
+ int[] rleCostSingle = new int[m];
+ int[] deCostSingle = new int[m];
+
+ int[] threshold = thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ for (int i = 0; i < m; i++) {
+ int currentValue = (x[0] >> i) & 1;
+ boolean hasOne = currentValue == 1;
+ int runCount = 1;
+ boolean changed = false;
+
+ for (int j = 1; j < xLength; j++) {
+ int subcolumnValue = (x[j] >> i) & 1;
+ if (subcolumnValue == 1) {
+ hasOne = true;
+ }
+ if (subcolumnValue != currentValue) {
+ runCount++;
+ currentValue = subcolumnValue;
+ changed = true;
+ }
+ }
+
+ bpeCostSingle[i] = hasOne ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = changed ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 0;
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 1;
+ cost1 += rleCostSingle[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = new int[l];
+ int mask = (1 << beta) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ int runCount = countGroupedRuns(x, xLength, groupStart, mask);
+ int rleCost = runCount * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount = countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, threshold[beta - 1]);
+ if (distinctCount < threshold[beta - 1]) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+
+ cost += currentCost;
+ }
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType) {
+ int listLength = list.length;
+ int maxValue = 0;
+ for (int value : list) {
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = new int[l];
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+ boolean[] seenValues = new boolean[mask + 1];
+ int[] dictKeyList = new int[mask + 1];
+ int[] codeMap = new int[mask + 1];
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+
+ if (encodingType[i] == 2) {
+ Arrays.fill(seenValues, false);
+ int cardinality = 0;
+ for (int j = 0; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (!seenValues[current]) {
+ seenValues[current] = true;
+ cardinality++;
+ }
+ }
+
+ int dictBitWidth = bitWidth(cardinality);
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if (seenValues[value]) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+ encodePos = bitPacking(subcolumnBuffer, bitWidthList[i], encodePos, encodedResult,
+ listLength);
+ } else {
+ int previous = subcolumnBuffer[0];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = subcolumnBuffer[j];
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(runLength, bw, encodePos, encodedResult, runCount);
+ encodePos = bitPacking(rleValues, bitWidthList[i], encodePos, encodedResult,
+ runCount);
+ }
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ int listLength = list.length;
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = new int[listLength];
+ int[] runLength = new int[listLength];
+ int[] rleValues = new int[listLength];
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+
+ if (type == 0) {
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ listLength, subcolumnBuffer);
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j];
+ while (currentIndex < endPos) {
+ subcolumnBuffer[currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = dictKeyList[subcolumnBuffer[j]];
+ }
+ }
+
+ int shiftAmount = i * beta;
+ for (int j = 0; j < listLength; j++) {
+ list[j] |= subcolumnBuffer[j] << shiftAmount;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining];
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ tsBlockDelta[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return tsBlockDelta;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ return BlockEncoder(data, blockIndex, blockSize, remainder, encodePos, encodedResult,
+ beta, null);
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta, EncodingTypeStats stats) {
+ int[] minDelta = new int[1];
+ int[] dataDelta = getAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta);
+
+ int2Bytes(minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (dataDelta[j] > maxValue) {
+ maxValue = dataDelta[j];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+ int[] encodingType = new int[m];
+ beta[0] = Subcolumn(dataDelta, remainder, m, blockSize, encodingType);
+ if (stats != null) {
+ int betaValue = beta[0];
+ int length = m == 0 ? 0 : (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = computeGroupedMaxBitWidths(dataDelta, remainder, m, betaValue);
+ stats.recordEncodingType(encodingType, length, betaValue, bitWidthList);
+ }
+
+ return SubcolumnEncoder(dataDelta, encodePos, encodedResult, beta, blockSize,
+ encodingType);
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int[] blockData = new int[remainder];
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, blockData, blockSize);
+
+ int base = blockIndex * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = blockData[i] + minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, null);
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult,
+ EncodingTypeStats stats) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {2};
+ if (stats != null) {
+ stats.setTotalBlockCount(numBlocks + (remainder > 0 ? 1 : 0));
+ }
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult,
+ beta, stats);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockEncoder(data, numBlocks, blockSize, remainder, encodePos,
+ encodedResult, beta, stats);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+
+ String outputParentDir = parentDir + "result/";
+ // String outputParentDir = "D://encoding-subcolumn/result/";
+ // String outputPath = outputParentDir + "subcolumn_encoding_type_ratio.csv";
+ String outputPath = outputParentDir + "subcolumn_encoding_type_ratio_2_32.csv";
+
+ int blockSize = 512;
+ blockSize = 32;
+
+ int repeatTime = 500;
+ repeatTime = 20;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ "Block Count",
+ "Subcolumn Count",
+ "Bit Packing Subcolumn Count",
+ "Bit Packing Ratio",
+ "RLE Subcolumn Count",
+ "RLE Ratio",
+ "Dictionary Subcolumn Count",
+ "Dictionary Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ System.out.println(maxDecimal);
+ byte[] encodedResult = new byte[data2Arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressedSize = 0;
+ int length = 0;
+ EncodingTypeStats stats = new EncodingTypeStats();
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ stats = new EncodingTypeStats();
+ length = Encoder(data2Arr, blockSize, encodedResult, stats);
+ }
+ long end = System.nanoTime();
+
+ encodeTime += ((end - start) / repeatTime);
+ compressedSize += length;
+
+ double ratioTmp = compressedSize / (double) (data1.size() * Long.BYTES);
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ end = System.nanoTime();
+ decodeTime += ((end - start) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio),
+ String.valueOf(stats.getTotalBlockCount()),
+ String.valueOf(stats.getTotalSubcolumnCount()),
+ String.valueOf(stats.getBitPackingSubcolumnCount()),
+ String.valueOf(stats.getBitPackingRatio()),
+ String.valueOf(stats.getRleSubcolumnCount()),
+ String.valueOf(stats.getRleRatio()),
+ String.valueOf(stats.getDictionarySubcolumnCount()),
+ String.valueOf(stats.getDictionaryRatio())
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ /** Diagnostic: print filter breakdown for Stocks-USA (run with -Dtest=...#diagnoseStocksUsaFilter). */
+ @Test
+ public void diagnoseStocksUsaFilter() throws IOException {
+ String path = "D://github/xjz17/subcolumn/dataset/Stocks-USA.csv";
+ InputStream inputStream = Files.newInputStream(new File(path).toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int cur = getDecimalPrecision(fStr);
+ if (cur > maxDecimal) {
+ maxDecimal = cur;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ int[] data = new int[data1.size()];
+ for (int i = 0; i < data1.size(); i++) {
+ data[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ int blockSize = 32;
+ int rawSlots = 0;
+ int skippedFullWidthBp = 0;
+ int[] rawType = new int[3];
+ int[] countedType = new int[3];
+ int[] bpByBw = new int[5]; // bw 0..4
+ int[] skipBpByBw = new int[5];
+ int[] betaPick = new int[5]; // [0]=unused, [1]=beta1, [2]=b2, ...
+
+ for (int bi = 0; bi + blockSize <= data.length; bi += blockSize) {
+ int[] minDelta = new int[1];
+ int[] delta = getAbsDeltaTsBlock(data, bi / blockSize, blockSize, blockSize, minDelta);
+ int maxValue = 0;
+ for (int v : delta) {
+ if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ int m = bitWidth(maxValue);
+ if (m == 0) {
+ continue;
+ }
+ int[] encodingType = new int[m];
+ int[] betaArr = new int[] {2};
+ betaArr[0] = Subcolumn(delta, blockSize, m, blockSize, encodingType);
+ int betaValue = betaArr[0];
+ if (betaValue >= 0 && betaValue < betaPick.length) {
+ betaPick[betaValue]++;
+ }
+ int length = m == 0 ? 0 : (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = computeGroupedMaxBitWidths(delta, blockSize, m, betaValue);
+ rawSlots += length;
+ for (int i = 0; i < length; i++) {
+ int t = encodingType[i];
+ int bw = bitWidthList[i];
+ if (t >= 0 && t < 3) {
+ rawType[t]++;
+ }
+ if (t == 0 && i < bitWidthList.length && bw == betaValue) {
+ skippedFullWidthBp++;
+ if (bw < skipBpByBw.length) {
+ skipBpByBw[bw]++;
+ }
+ continue;
+ }
+ if (t >= 0 && t < 3) {
+ countedType[t]++;
+ }
+ if (t == 0 && bw < bpByBw.length) {
+ bpByBw[bw]++;
+ }
+ }
+ }
+
+ int countedTotal = countedType[0] + countedType[1] + countedType[2];
+ System.out.println("=== Stocks-USA filter diagnosis (blockSize=32) ===");
+ System.out.println("raw grouped subcolumn slots: " + rawSlots);
+ System.out.println(
+ "before filter: BPE=" + rawType[0] + " RLE=" + rawType[1] + " Dict=" + rawType[2]);
+ System.out.println("skipped (type=0 && bitWidth==beta): " + skippedFullWidthBp);
+ System.out.println(
+ "after filter: BPE=" + countedType[0] + " RLE=" + countedType[1]
+ + " Dict=" + countedType[2] + " total=" + countedTotal);
+ System.out.println(
+ "BPE ratio=" + (countedTotal == 0 ? 0 : countedType[0] / (double) countedTotal));
+ System.out.println(
+ "counted BPE by bitWidth: bw0=" + bpByBw[0] + " bw1=" + bpByBw[1]
+ + " bw2=" + bpByBw[2] + " bw3=" + bpByBw[3]);
+ System.out.println(
+ "skipped BPE by bitWidth: bw0=" + skipBpByBw[0] + " bw1=" + skipBpByBw[1]
+ + " bw2=" + skipBpByBw[2] + " bw3=" + skipBpByBw[3]);
+ System.out.println(
+ "beta picked: b1=" + betaPick[1] + " b2=" + betaPick[2] + " b3=" + betaPick[3]
+ + " b4=" + betaPick[4]);
+ System.out.println(
+ "NOTE: beta=1 means per-bit (length=m); filter bw==1 excludes almost all BPE.");
+ }
+
+}
\ No newline at end of file
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewTest.java
new file mode 100644
index 0000000..a4de7c6
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneNewTest.java
@@ -0,0 +1,2009 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+public class SubcolumnPruneNewTest {
+
+ private static final ThreadLocal<EncodeScratch> ENCODE_SCRATCH =
+ ThreadLocal.withInitial(EncodeScratch::new);
+
+ private static final ThreadLocal<DecodeScratch> DECODE_SCRATCH =
+ ThreadLocal.withInitial(DecodeScratch::new);
+
+ private static final class EncodeScratch {
+ private final int[] dataDelta = new int[8192];
+ private final int[] bpeCostSingle = new int[32];
+ private final int[] rleCostSingle = new int[32];
+ private final int[] deCostSingle = new int[32];
+ private final int[] encodingType = new int[32];
+ private final int[] encodingTypeTemp = new int[32];
+ private final int[] bitWidthList = new int[32];
+ private final int[] subcolumnBuffer = new int[8192];
+ private final int[] runLength = new int[8192];
+ private final int[] rleValues = new int[8192];
+ private final int[] dictKeyList = new int[16];
+ private final int[] codeMap = new int[16];
+ private final int[] minDelta = new int[1];
+ private final int[] minDelta3 = new int[3];
+ private final int[] beta = new int[1];
+ private final int[] betaCandidateOrder = new int[3];
+ private final int[] betaCandidateLowerBound = new int[3];
+ private final int[] betaCandidateCost = new int[3];
+ private final int[] betaCandidateEncodingType = new int[3 * 32];
+ private int lastBestBeta = 2;
+ /** Flattened grouped subcolumns: group i starts at i * listLength. */
+ private final int[] groupFlat = new int[32 * 8192];
+ private final int[] groupMax = new int[32];
+ private int cachedBeta = -1;
+ private int cachedL;
+ private int cachedListLength = -1;
+ }
+
+ private static final class DecodeScratch {
+ private int[] bitWidthList = new int[32];
+ private int[] encodingType = new int[32];
+ private int[] subcolumnBuffer = new int[8192];
+ private int[] runLength = new int[8192];
+ private int[] rleValues = new int[8192];
+ private int[] dictKeyList = new int[16];
+
+ private void ensureL(int l) {
+ if (bitWidthList.length < l) {
+ bitWidthList = new int[l];
+ encodingType = new int[l];
+ }
+ }
+
+ private void ensureListLength(int listLength) {
+ if (subcolumnBuffer.length < listLength) {
+ subcolumnBuffer = new int[listLength];
+ runLength = new int[listLength];
+ rleValues = new int[listLength];
+ }
+ }
+
+ private int[] ensureDict(int cardinality) {
+ if (dictKeyList.length < cardinality) {
+ dictKeyList = new int[cardinality];
+ }
+ return dictKeyList;
+ }
+ }
+
+ private static final int[] DEFAULT_THRESHOLD =
+ {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ private static final int[] THRESHOLD_64 =
+ {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ private static final int[] THRESHOLD_128 =
+ {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ private static final int[] THRESHOLD_256 =
+ {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ private static final int[] THRESHOLD_512 =
+ {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ private static final int[] THRESHOLD_1024 =
+ {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ private static final int[] THRESHOLD_2048 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ private static final int[] THRESHOLD_4096 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ private static final int[] THRESHOLD_8192 =
+ {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ private static final int[] BETA_LIST = {2, 3, 4};
+ private static boolean USE_ALPHA_HYBRID = false;
+ private static boolean USE_ALPHA_FAST_HYBRID = false;
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encodePos,
+ byte[] encodedResult) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encodedResult[encodePos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encodePos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] resultList,
+ int resultOffset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ resultList[resultOffset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ return bitPackingAt(numbers, 0, bitWidth, encodePos, encodedResult, numValues);
+ }
+
+ private static int bitPackingAt(int[] numbers, int offset, int bitWidth, int encodePos,
+ byte[] encodedResult, int numValues) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8At(numbers, offset, encodePos, encodedResult, numValues);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ pack8Values(numbers, offset + i * 8, bitWidth, encodePos, encodedResult);
+ encodePos += bitWidth;
+ }
+
+ encodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[offset + blockNum * 8 + i], encodedResult, encodePos, bitWidth);
+ encodePos += bitWidth;
+ }
+
+ return (encodePos + 7) / 8;
+ }
+
+ private static int bitPackingWidth1At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 7) | (numbers[base + 1] << 6)
+ | (numbers[base + 2] << 5) | (numbers[base + 3] << 4)
+ | (numbers[base + 4] << 3) | (numbers[base + 5] << 2)
+ | (numbers[base + 6] << 1) | numbers[base + 7]);
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth2At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 6) | (numbers[base + 1] << 4)
+ | (numbers[base + 2] << 2) | numbers[base + 3]);
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < numValues) {
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth4At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int base = offset + i;
+ encodedResult[encodePos] = (byte) ((numbers[base] << 4) | numbers[base + 1]);
+ encodePos++;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = encodePos * 8;
+ intToBytes(numbers[offset + i], encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingWidth8At(int[] numbers, int offset, int encodePos,
+ byte[] encodedResult, int numValues) {
+ for (int i = 0; i < numValues; i++) {
+ encodedResult[encodePos++] = (byte) numbers[offset + i];
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingShifted(int[] list, int listLength, int shiftAmount, int mask,
+ int bitWidth, int encodePos, byte[] encodedResult, int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return encodePos;
+ }
+ if (bitWidth == 1) {
+ return bitPackingWidth1Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 2) {
+ return bitPackingWidth2Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 4) {
+ return bitPackingWidth4Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+ if (bitWidth == 8) {
+ return bitPackingWidth8Shifted(list, listLength, shiftAmount, mask, encodePos,
+ encodedResult);
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ fallbackBuffer[j] = (list[j] >> shiftAmount) & mask;
+ }
+ return bitPacking(fallbackBuffer, bitWidth, encodePos, encodedResult, listLength);
+ }
+
+ private static int bitPackingWidth1Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 8 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 7)
+ | (((list[i + 1] >> shiftAmount) & mask) << 6)
+ | (((list[i + 2] >> shiftAmount) & mask) << 5)
+ | (((list[i + 3] >> shiftAmount) & mask) << 4)
+ | (((list[i + 4] >> shiftAmount) & mask) << 3)
+ | (((list[i + 5] >> shiftAmount) & mask) << 2)
+ | (((list[i + 6] >> shiftAmount) & mask) << 1)
+ | ((list[i + 7] >> shiftAmount) & mask));
+ encodePos++;
+ i += 8;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 1);
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth2Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 4 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 6)
+ | (((list[i + 1] >> shiftAmount) & mask) << 4)
+ | (((list[i + 2] >> shiftAmount) & mask) << 2)
+ | ((list[i + 3] >> shiftAmount) & mask));
+ encodePos++;
+ i += 4;
+ }
+ int bitPos = encodePos * 8;
+ while (i < listLength) {
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 2);
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int bitPackingWidth4Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ int i = 0;
+ while (i + 2 <= listLength) {
+ encodedResult[encodePos] = (byte) ((((list[i] >> shiftAmount) & mask) << 4)
+ | ((list[i + 1] >> shiftAmount) & mask));
+ encodePos++;
+ i += 2;
+ }
+ if (i < listLength) {
+ int bitPos = encodePos * 8;
+ intToBytes((list[i] >> shiftAmount) & mask, encodedResult, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return encodePos;
+ }
+
+ private static int bitPackingWidth8Shifted(int[] list, int listLength, int shiftAmount,
+ int mask, int encodePos, byte[] encodedResult) {
+ for (int i = 0; i < listLength; i++) {
+ encodedResult[encodePos++] = (byte) ((list[i] >> shiftAmount) & mask);
+ }
+ return encodePos;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decodePos, int bitWidth, int numValues, int[] resultList) {
+ if (bitWidth == 0) {
+ Arrays.fill(resultList, 0, numValues, 0);
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4(encoded, decodePos, numValues, resultList);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8(encoded, decodePos, numValues, resultList);
+ }
+
+ int blockNum = numValues / 8;
+ int remainder = numValues % 8;
+
+ for (int i = 0; i < blockNum; i++) {
+ unpack8Values(encoded, decodePos, bitWidth, resultList, i * 8);
+ decodePos += bitWidth;
+ }
+
+ decodePos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ resultList[blockNum * 8 + i] = bytesToInt(encoded, decodePos, bitWidth);
+ decodePos += bitWidth;
+ }
+
+ return (decodePos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth1(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 7) & 1;
+ resultList[i + 1] = (value >>> 6) & 1;
+ resultList[i + 2] = (value >>> 5) & 1;
+ resultList[i + 3] = (value >>> 4) & 1;
+ resultList[i + 4] = (value >>> 3) & 1;
+ resultList[i + 5] = (value >>> 2) & 1;
+ resultList[i + 6] = (value >>> 1) & 1;
+ resultList[i + 7] = value & 1;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 1);
+ bitPos++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth2(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 6) & 3;
+ resultList[i + 1] = (value >>> 4) & 3;
+ resultList[i + 2] = (value >>> 2) & 3;
+ resultList[i + 3] = value & 3;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ resultList[i++] = bytesToInt(encoded, bitPos, 2);
+ bitPos += 2;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth4(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ resultList[i] = (value >>> 4) & 15;
+ resultList[i + 1] = value & 15;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ resultList[i] = bytesToInt(encoded, bitPos, 4);
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth8(
+ byte[] encoded, int decodePos, int numValues, int[] resultList) {
+ for (int i = 0; i < numValues; i++) {
+ resultList[i] = encoded[decodePos++] & 0xFF;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingOrShifted(
+ byte[] encoded,
+ int decodePos,
+ int bitWidth,
+ int numValues,
+ int[] output,
+ int outputOffset,
+ int shiftAmount,
+ int[] fallbackBuffer) {
+ if (bitWidth == 0) {
+ return decodePos;
+ }
+ if (bitWidth == 1) {
+ return decodeBitPackingWidth1OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 2) {
+ return decodeBitPackingWidth2OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 4) {
+ return decodeBitPackingWidth4OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+ if (bitWidth == 8) {
+ return decodeBitPackingWidth8OrShifted(encoded, decodePos, numValues, output,
+ outputOffset, shiftAmount);
+ }
+
+ decodePos = decodeBitPacking(encoded, decodePos, bitWidth, numValues, fallbackBuffer);
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= fallbackBuffer[i] << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth1OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 8 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 7) & 1) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 6) & 1) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 5) & 1) << shiftAmount;
+ output[outputOffset + i + 3] |= ((value >>> 4) & 1) << shiftAmount;
+ output[outputOffset + i + 4] |= ((value >>> 3) & 1) << shiftAmount;
+ output[outputOffset + i + 5] |= ((value >>> 2) & 1) << shiftAmount;
+ output[outputOffset + i + 6] |= ((value >>> 1) & 1) << shiftAmount;
+ output[outputOffset + i + 7] |= (value & 1) << shiftAmount;
+ i += 8;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 1) << shiftAmount;
+ bitPos++;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth2OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 4 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 6) & 3) << shiftAmount;
+ output[outputOffset + i + 1] |= ((value >>> 4) & 3) << shiftAmount;
+ output[outputOffset + i + 2] |= ((value >>> 2) & 3) << shiftAmount;
+ output[outputOffset + i + 3] |= (value & 3) << shiftAmount;
+ i += 4;
+ }
+ int bitPos = decodePos * 8;
+ while (i < numValues) {
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 2) << shiftAmount;
+ bitPos += 2;
+ i++;
+ }
+ return (bitPos + 7) / 8;
+ }
+
+ private static int decodeBitPackingWidth4OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ int i = 0;
+ while (i + 2 <= numValues) {
+ int value = encoded[decodePos++] & 0xFF;
+ output[outputOffset + i] |= ((value >>> 4) & 15) << shiftAmount;
+ output[outputOffset + i + 1] |= (value & 15) << shiftAmount;
+ i += 2;
+ }
+ if (i < numValues) {
+ int bitPos = decodePos * 8;
+ output[outputOffset + i] |= bytesToInt(encoded, bitPos, 4) << shiftAmount;
+ bitPos += 4;
+ return (bitPos + 7) / 8;
+ }
+ return decodePos;
+ }
+
+ private static int decodeBitPackingWidth8OrShifted(
+ byte[] encoded, int decodePos, int numValues, int[] output, int outputOffset,
+ int shiftAmount) {
+ for (int i = 0; i < numValues; i++) {
+ output[outputOffset + i] |= (encoded[decodePos++] & 0xFF) << shiftAmount;
+ }
+ return decodePos;
+ }
+
+ public static void int2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static void intByte2Bytes(int integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) integer;
+ }
+
+ public static void long2intBytes(long integer, int encodePos, byte[] currentBytes) {
+ currentBytes[encodePos] = (byte) (integer >> 24);
+ currentBytes[encodePos + 1] = (byte) (integer >> 16);
+ currentBytes[encodePos + 2] = (byte) (integer >> 8);
+ currentBytes[encodePos + 3] = (byte) integer;
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decodePos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decodePos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int[] thresholdForBlockSize(int blockSize) {
+ switch (blockSize) {
+ case 64:
+ return THRESHOLD_64;
+ case 128:
+ return THRESHOLD_128;
+ case 256:
+ return THRESHOLD_256;
+ case 512:
+ return THRESHOLD_512;
+ case 1024:
+ return THRESHOLD_1024;
+ case 2048:
+ return THRESHOLD_2048;
+ case 4096:
+ return THRESHOLD_4096;
+ case 8192:
+ return THRESHOLD_8192;
+ case 32:
+ default:
+ return DEFAULT_THRESHOLD;
+ }
+ }
+
+ private static int countGroupedRuns(int[] values, int length, int shiftAmount, int mask) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int runs = 1;
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ }
+ return runs;
+ }
+
+ private static int countDistinctValuesUntilLimit(int[] values, int length, int shiftAmount,
+ int mask, int limit) {
+ int seenMask = 0;
+ int distinctCount = 0;
+ for (int i = 0; i < length; i++) {
+ int value = (values[i] >> shiftAmount) & mask;
+ int bit = 1 << value;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ if (distinctCount >= limit) {
+ return distinctCount;
+ }
+ }
+ }
+ return distinctCount;
+ }
+
+ private static int countGroupedRunsAndDistinctUntilLimit(
+ int[] values,
+ int length,
+ int shiftAmount,
+ int mask,
+ int distinctLimit,
+ int[] out) {
+ int previous = (values[0] >> shiftAmount) & mask;
+ int seenMask = 1 << previous;
+ int runs = 1;
+ int distinctCount = 1;
+
+ for (int i = 1; i < length; i++) {
+ int current = (values[i] >> shiftAmount) & mask;
+ if (current != previous) {
+ runs++;
+ previous = current;
+ }
+ int bit = 1 << current;
+ if ((seenMask & bit) == 0) {
+ seenMask |= bit;
+ distinctCount++;
+ }
+ }
+
+ out[0] = runs;
+ out[1] = distinctCount >= distinctLimit ? distinctLimit : distinctCount;
+ return distinctCount;
+ }
+
+ private static int betaLowerBound(
+ int beta,
+ int m,
+ int xLength,
+ int[] bpeCostSingle,
+ int[] rleCostSingle,
+ int[] deCostSingle) {
+ int l = (m + beta - 1) / beta;
+ int lowerBound = 0;
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int bpeCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+ int rleCostMax = 0;
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+ lowerBound += Math.min(bpeCost, Math.min(rleCostMax, deCostMax));
+ }
+ return lowerBound;
+ }
+
+ private static int buildHybridBetaOrder(
+ int m,
+ int xLength,
+ int[] bpeCostSingle,
+ int[] rleCostSingle,
+ int[] deCostSingle,
+ EncodeScratch scratch) {
+ int count = 0;
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+ scratch.betaCandidateOrder[count] = beta;
+ scratch.betaCandidateLowerBound[count] =
+ betaLowerBound(beta, m, xLength, bpeCostSingle, rleCostSingle, deCostSingle);
+ count++;
+ }
+
+ for (int i = 0; i < count; i++) {
+ for (int j = i + 1; j < count; j++) {
+ boolean preferJ = scratch.betaCandidateLowerBound[j]
+ < scratch.betaCandidateLowerBound[i];
+ if (scratch.betaCandidateOrder[j] == scratch.lastBestBeta
+ && scratch.betaCandidateOrder[i] != scratch.lastBestBeta) {
+ preferJ = true;
+ }
+ if (preferJ) {
+ int betaTmp = scratch.betaCandidateOrder[i];
+ scratch.betaCandidateOrder[i] = scratch.betaCandidateOrder[j];
+ scratch.betaCandidateOrder[j] = betaTmp;
+
+ int lowerTmp = scratch.betaCandidateLowerBound[i];
+ scratch.betaCandidateLowerBound[i] = scratch.betaCandidateLowerBound[j];
+ scratch.betaCandidateLowerBound[j] = lowerTmp;
+ }
+ }
+ }
+ return count;
+ }
+
+ private static void extractAllGroups(
+ EncodeScratch scratch,
+ int[] x,
+ int xLength,
+ int beta,
+ int m,
+ int mask) {
+ int l = (m + beta - 1) / beta;
+ int[] flat = scratch.groupFlat;
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ int maxValuePart = 0;
+ int base = i * xLength;
+ for (int j = 0; j < xLength; j++) {
+ int current = (x[j] >> shiftAmount) & mask;
+ flat[base + j] = current;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ scratch.groupMax[i] = maxValuePart;
+ }
+ scratch.cachedBeta = beta;
+ scratch.cachedL = l;
+ scratch.cachedListLength = xLength;
+ }
+
+ private static boolean useGroupCache(EncodeScratch scratch, int betaValue, int l, int listLength) {
+ return scratch.cachedBeta == betaValue
+ && scratch.cachedL == l
+ && scratch.cachedListLength == listLength;
+ }
+
+ public static int Subcolumn(int[] x, int xLength, int m, int blockSize, int[] encodingType) {
+ return Subcolumn(x, xLength, m, blockSize, encodingType, ENCODE_SCRATCH.get());
+ }
+
+ private static int Subcolumn(
+ int[] x,
+ int xLength,
+ int m,
+ int blockSize,
+ int[] encodingType,
+ EncodeScratch scratch) {
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+ int[] bpeCostSingle = scratch.bpeCostSingle;
+ int[] rleCostSingle = scratch.rleCostSingle;
+ int[] deCostSingle = scratch.deCostSingle;
+ int[] encodingTypeTemp = scratch.encodingTypeTemp;
+ int[] groupStats = scratch.minDelta3;
+
+ int[] threshold = blockSize == 512 ? THRESHOLD_512 : thresholdForBlockSize(blockSize);
+ int lengthBitWidth = bitWidth(xLength);
+ int cost1 = 0;
+
+ Arrays.fill(rleCostSingle, 0, m, 1);
+ int valueMask = m == Integer.SIZE ? -1 : (1 << m) - 1;
+ int previousValue = x[0] & valueMask;
+ int unionValue = previousValue;
+ for (int j = 1; j < xLength; j++) {
+ int currentValue = x[j] & valueMask;
+ unionValue |= currentValue;
+ int changedBits = previousValue ^ currentValue;
+ while (changedBits != 0) {
+ int changedBit = Integer.numberOfTrailingZeros(changedBits);
+ rleCostSingle[changedBit]++;
+ changedBits &= changedBits - 1;
+ }
+ previousValue = currentValue;
+ }
+
+ for (int i = 0; i < m; i++) {
+ int runCount = rleCostSingle[i];
+
+ bpeCostSingle[i] = ((unionValue >>> i) & 1) == 1 ? xLength : 0;
+ rleCostSingle[i] = runCount * (1 + lengthBitWidth);
+ deCostSingle[i] = runCount > 1 ? xLength * 2 + 2 : xLength + 2;
+
+ if (bpeCostSingle[i] <= rleCostSingle[i] && bpeCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 0;
+ cost1 += bpeCostSingle[i];
+ } else if (rleCostSingle[i] < bpeCostSingle[i] && rleCostSingle[i] <= deCostSingle[i]) {
+ encodingType[i] = 1;
+ cost1 += rleCostSingle[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += deCostSingle[i];
+ }
+ }
+
+ int cMin = cost1;
+
+ int betaCandidateCount = USE_ALPHA_HYBRID
+ ? buildHybridBetaOrder(m, xLength, bpeCostSingle, rleCostSingle, deCostSingle,
+ scratch)
+ : BETA_LIST.length;
+ if (USE_ALPHA_HYBRID) {
+ Arrays.fill(scratch.betaCandidateCost, Integer.MAX_VALUE);
+ }
+ for (int betaCandidateIndex = 0; betaCandidateIndex < betaCandidateCount;
+ betaCandidateIndex++) {
+ int beta = USE_ALPHA_HYBRID
+ ? scratch.betaCandidateOrder[betaCandidateIndex]
+ : BETA_LIST[betaCandidateIndex];
+ if (beta > m) {
+ break;
+ }
+ if (USE_ALPHA_FAST_HYBRID
+ && scratch.betaCandidateLowerBound[betaCandidateIndex] >= cMin) {
+ continue;
+ }
+
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int mask = (1 << beta) - 1;
+ int betaThreshold = threshold[beta - 1];
+ for (int t = 0; t < l; t++) {
+ encodingTypeTemp[t] = 0;
+ }
+
+ for (int i = 0; i < l; i++) {
+ int groupStart = i * beta;
+ int groupEnd = Math.min(m, groupStart + beta);
+ int betaStart = groupEnd - 1;
+
+ while (betaStart >= groupStart && bpeCostSingle[betaStart] == 0) {
+ betaStart--;
+ }
+
+ if (betaStart < groupStart) {
+ betaStart = groupStart;
+ }
+
+ int currentCost = bpeCostSingle[betaStart] * (betaStart - groupStart + 1);
+
+ int rleCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (rleCostSingle[j] > rleCostMax) {
+ rleCostMax = rleCostSingle[j];
+ }
+ }
+
+ int deCostMax = 0;
+ for (int j = groupStart; j < groupEnd; j++) {
+ if (deCostSingle[j] > deCostMax) {
+ deCostMax = deCostSingle[j];
+ }
+ }
+
+ boolean needRle = rleCostMax < currentCost;
+ boolean maybeNeedDe = deCostMax < currentCost;
+ int groupedRunCount = -1;
+ int groupedDistinctCount = -1;
+
+ if (needRle && maybeNeedDe) {
+ countGroupedRunsAndDistinctUntilLimit(
+ x, xLength, groupStart, mask, betaThreshold, groupStats);
+ groupedRunCount = groupStats[0];
+ groupedDistinctCount = groupStats[1];
+ }
+
+ if (needRle) {
+ int runCount = groupedRunCount >= 0
+ ? groupedRunCount
+ : countGroupedRuns(x, xLength, groupStart, mask);
+ int rleCost = runCount * (beta + lengthBitWidth);
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ int distinctCount = groupedDistinctCount >= 0
+ ? groupedDistinctCount
+ : countDistinctValuesUntilLimit(
+ x, xLength, groupStart, mask, betaThreshold);
+ if (distinctCount < betaThreshold) {
+ int deCost = xLength * bitWidth(distinctCount) + distinctCount * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+
+ cost += currentCost;
+ int pruningLimit = USE_ALPHA_FAST_HYBRID ? cMin : (USE_ALPHA_HYBRID ? cost1 : cMin);
+ if (cost >= pruningLimit) {
+ break;
+ }
+ }
+
+ if (USE_ALPHA_FAST_HYBRID) {
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ } else if (USE_ALPHA_HYBRID) {
+ int candidateIndex = beta - BETA_LIST[0];
+ scratch.betaCandidateCost[candidateIndex] = cost;
+ if (cost < cost1) {
+ System.arraycopy(
+ encodingTypeTemp,
+ 0,
+ scratch.betaCandidateEncodingType,
+ candidateIndex * 32,
+ l);
+ }
+ } else if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ if (USE_ALPHA_HYBRID && !USE_ALPHA_FAST_HYBRID) {
+ cMin = cost1;
+ betaBest = 1;
+ for (int beta : BETA_LIST) {
+ if (beta > m) {
+ break;
+ }
+ int candidateIndex = beta - BETA_LIST[0];
+ int cost = scratch.betaCandidateCost[candidateIndex];
+ if (cost < cMin) {
+ int l = (m + beta - 1) / beta;
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(
+ scratch.betaCandidateEncodingType,
+ candidateIndex * 32,
+ encodingType,
+ 0,
+ l);
+ }
+ }
+ }
+
+ if (betaBest > 1) {
+ extractAllGroups(scratch, x, xLength, betaBest, m, (1 << betaBest) - 1);
+ }
+ scratch.lastBestBeta = betaBest;
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType) {
+ return SubcolumnEncoder(list, list.length, encodePos, encodedResult, beta, blockSize,
+ encodingType, -1);
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encodePos, byte[] encodedResult,
+ int[] beta, int blockSize, int[] encodingType, int knownM) {
+ return SubcolumnEncoder(list, list.length, encodePos, encodedResult, beta, blockSize,
+ encodingType, knownM);
+ }
+
+ public static int SubcolumnEncoder(
+ int[] list,
+ int listLength,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ int blockSize,
+ int[] encodingType,
+ int knownM) {
+ int m = knownM;
+ if (m < 0) {
+ int maxValue = 0;
+ for (int i = 0; i < listLength; i++) {
+ int value = list[i];
+ if (value > maxValue) {
+ maxValue = value;
+ }
+ }
+ m = bitWidth(maxValue);
+ }
+ return SubcolumnEncoder(list, listLength, encodePos, encodedResult, beta, blockSize,
+ encodingType, m, ENCODE_SCRATCH.get());
+ }
+
+ private static int encodeRleRuns(
+ int[] runLength,
+ int[] rleValues,
+ int runCount,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ encodedResult[encodePos] = (byte) (runCount >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (runCount & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(runLength, runLengthBitWidth, encodePos, encodedResult, runCount);
+ return bitPacking(rleValues, valueBitWidth, encodePos, encodedResult, runCount);
+ }
+
+ private static int encodeRleFromValues(
+ int[] values,
+ int offset,
+ int listLength,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = values[offset];
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = values[offset + j];
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ private static int encodeRleShifted(
+ int[] list,
+ int listLength,
+ int shiftAmount,
+ int mask,
+ int[] runLength,
+ int[] rleValues,
+ int runLengthBitWidth,
+ int valueBitWidth,
+ int encodePos,
+ byte[] encodedResult) {
+ int previous = (list[0] >> shiftAmount) & mask;
+ int runCount = 0;
+
+ for (int j = 1; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current != previous) {
+ runLength[runCount] = j;
+ rleValues[runCount] = previous;
+ runCount++;
+ previous = current;
+ }
+ }
+
+ runLength[runCount] = listLength;
+ rleValues[runCount] = previous;
+ runCount++;
+
+ return encodeRleRuns(runLength, rleValues, runCount, runLengthBitWidth, valueBitWidth,
+ encodePos, encodedResult);
+ }
+
+ public static int[] borrowMinDelta3Buffer() {
+ return ENCODE_SCRATCH.get().minDelta3;
+ }
+
+ public static int[] borrowEncodingTypeBuffer() {
+ return ENCODE_SCRATCH.get().encodingType;
+ }
+
+ public static int[] borrowDataDeltaBuffer() {
+ return ENCODE_SCRATCH.get().dataDelta;
+ }
+
+ private static int SubcolumnEncoder(
+ int[] list,
+ int listLength,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ int blockSize,
+ int[] encodingType,
+ int m,
+ EncodeScratch scratch) {
+ intByte2Bytes(m, encodePos, encodedResult);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int betaValue = beta[0];
+ int l = (m + betaValue - 1) / betaValue;
+ int[] bitWidthList = scratch.bitWidthList;
+ int[] subcolumnBuffer = scratch.subcolumnBuffer;
+ int[] runLength = scratch.runLength;
+ int[] rleValues = scratch.rleValues;
+ int[] dictKeyList = scratch.dictKeyList;
+ int[] codeMap = scratch.codeMap;
+
+ intByte2Bytes(betaValue, encodePos, encodedResult);
+ encodePos += 1;
+
+ int bw = bitWidth(blockSize);
+ int mask = (1 << betaValue) - 1;
+ boolean useCache = useGroupCache(scratch, betaValue, l, listLength);
+
+ if (useCache) {
+ for (int i = 0; i < l; i++) {
+ bitWidthList[i] = bitWidth(scratch.groupMax[i]);
+ }
+ } else if (betaValue == 1) {
+ int unionValue = 0;
+ for (int j = 0; j < listLength; j++) {
+ unionValue |= list[j];
+ }
+ for (int i = 0; i < l; i++) {
+ bitWidthList[i] = (unionValue >>> i) & 1;
+ }
+ } else {
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ int maxValuePart = 0;
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ if (current > maxValuePart) {
+ maxValuePart = current;
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+ }
+
+ encodePos = bitPacking(bitWidthList, 8, encodePos, encodedResult, l);
+
+ int preTypePos = encodePos;
+ encodePos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * betaValue;
+ int groupOffset = i * listLength;
+
+ if (encodingType[i] == 0) {
+ if (useCache) {
+ encodePos = bitPackingAt(scratch.groupFlat, groupOffset, bitWidthList[i],
+ encodePos, encodedResult, listLength);
+ } else {
+ encodePos = bitPackingShifted(list, listLength, shiftAmount, mask,
+ bitWidthList[i], encodePos, encodedResult, subcolumnBuffer);
+ }
+ continue;
+ }
+
+ if (encodingType[i] == 1) {
+ if (useCache) {
+ encodePos = encodeRleFromValues(scratch.groupFlat, groupOffset, listLength,
+ runLength, rleValues, bw, bitWidthList[i], encodePos, encodedResult);
+ } else {
+ encodePos = encodeRleShifted(list, listLength, shiftAmount, mask, runLength,
+ rleValues, bw, bitWidthList[i], encodePos, encodedResult);
+ }
+ continue;
+ }
+
+ int seenMask = 0;
+ if (useCache) {
+ System.arraycopy(scratch.groupFlat, groupOffset, subcolumnBuffer, 0, listLength);
+ for (int j = 0; j < listLength; j++) {
+ seenMask |= 1 << subcolumnBuffer[j];
+ }
+ } else {
+ for (int j = 0; j < listLength; j++) {
+ int current = (list[j] >> shiftAmount) & mask;
+ subcolumnBuffer[j] = current;
+ seenMask |= 1 << current;
+ }
+ }
+
+ int cardinality = Integer.bitCount(seenMask);
+ int dictBitWidth = bitWidth(cardinality);
+ int dictSize = 0;
+ for (int value = 0; value <= mask; value++) {
+ if ((seenMask & (1 << value)) != 0) {
+ dictKeyList[dictSize] = value;
+ codeMap[value] = dictSize;
+ dictSize++;
+ }
+ }
+
+ for (int j = 0; j < listLength; j++) {
+ subcolumnBuffer[j] = codeMap[subcolumnBuffer[j]];
+ }
+
+ encodedResult[encodePos] = (byte) (cardinality >> 8);
+ encodePos += 1;
+ encodedResult[encodePos] = (byte) (cardinality & 0xFF);
+ encodePos += 1;
+
+ encodePos = bitPacking(dictKeyList, bitWidthList[i], encodePos, encodedResult,
+ cardinality);
+ encodePos = bitPacking(subcolumnBuffer, dictBitWidth, encodePos, encodedResult,
+ listLength);
+ }
+
+ bitPacking(encodingType, 2, preTypePos, encodedResult, l);
+ return encodePos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int blockSize) {
+ return SubcolumnDecoder(encodedResult, encodePos, list, 0, list.length, blockSize);
+ }
+
+ private static int SubcolumnDecoder(byte[] encodedResult, int encodePos, int[] list,
+ int outputOffset, int listLength, int blockSize) {
+ int m = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ if (m == 0) {
+ return encodePos;
+ }
+
+ int bw = bitWidth(blockSize);
+ int beta = bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ DecodeScratch scratch = DECODE_SCRATCH.get();
+ scratch.ensureL(l);
+ scratch.ensureListLength(listLength);
+
+ int[] bitWidthList = scratch.bitWidthList;
+ encodePos = decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = scratch.encodingType;
+ encodePos = decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int[] subcolumnBuffer = scratch.subcolumnBuffer;
+ int[] runLength = scratch.runLength;
+ int[] rleValues = scratch.rleValues;
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+ int shiftAmount = i * beta;
+
+ if (type == 0) {
+ encodePos = decodeBitPackingOrShifted(encodedResult, encodePos, currentBitWidth,
+ listLength, list, outputOffset, shiftAmount, subcolumnBuffer);
+ continue;
+ } else if (type == 1) {
+ int index = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ encodePos = decodeBitPacking(encodedResult, encodePos, bw, index, runLength);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth, index,
+ rleValues);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = runLength[j];
+ int value = rleValues[j] << shiftAmount;
+ int outputBase = outputOffset + currentIndex;
+ for (int k = currentIndex; k < endPos; k++) {
+ list[outputBase++] |= value;
+ }
+ currentIndex = endPos;
+ }
+ continue;
+ } else {
+ int cardinality = ((encodedResult[encodePos] & 0xFF) << 8)
+ | (encodedResult[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ int dictBitWidth = bitWidth(cardinality);
+ int[] dictKeyList = scratch.ensureDict(cardinality);
+ encodePos = decodeBitPacking(encodedResult, encodePos, currentBitWidth,
+ cardinality, dictKeyList);
+ encodePos = decodeBitPacking(encodedResult, encodePos, dictBitWidth, listLength,
+ subcolumnBuffer);
+
+ for (int j = 0; j < listLength; j++) {
+ list[outputOffset + j] |= dictKeyList[subcolumnBuffer[j]] << shiftAmount;
+ }
+ continue;
+ }
+ }
+
+ return encodePos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(int[] tsBlock, int blockIndex, int blockSize,
+ int remaining, int[] minDelta) {
+ int[] out = new int[remaining];
+ fillAbsDeltaTsBlock(tsBlock, blockIndex, blockSize, remaining, minDelta, out);
+ return out;
+ }
+
+ private static int fillAbsDeltaTsBlock(
+ int[] tsBlock,
+ int blockIndex,
+ int blockSize,
+ int remaining,
+ int[] minDelta,
+ int[] out) {
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int valueDeltaMax = Integer.MIN_VALUE;
+ int base = blockIndex * blockSize;
+ int end = base + remaining;
+
+ for (int j = base; j < end; j++) {
+ int current = tsBlock[j];
+ if (current < valueDeltaMin) {
+ valueDeltaMin = current;
+ }
+ if (current > valueDeltaMax) {
+ valueDeltaMax = current;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ out[j - base] = tsBlock[j] - valueDeltaMin;
+ }
+
+ minDelta[0] = valueDeltaMin;
+ return valueDeltaMax - valueDeltaMin;
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta) {
+ return BlockEncoder(data, blockIndex, blockSize, remainder, encodePos, encodedResult, beta,
+ null, null);
+ }
+
+ public static int BlockEncoder(int[] data, int blockIndex, int blockSize, int remainder,
+ int encodePos, byte[] encodedResult, int[] beta, long[] forTime,
+ long[] subcolumnTime) {
+ EncodeScratch scratch = ENCODE_SCRATCH.get();
+ long forStart = System.nanoTime();
+ int maxValue = fillAbsDeltaTsBlock(
+ data, blockIndex, blockSize, remainder, scratch.minDelta, scratch.dataDelta);
+ long forEnd = System.nanoTime();
+ if (forTime != null) {
+ forTime[0] += (forEnd - forStart);
+ }
+
+ long subStart = System.nanoTime();
+ int2Bytes(scratch.minDelta[0], encodePos, encodedResult);
+ encodePos += 4;
+
+ int m = bitWidth(maxValue);
+ beta[0] = Subcolumn(scratch.dataDelta, remainder, m, blockSize, scratch.encodingType, scratch);
+ encodePos = SubcolumnEncoder(scratch.dataDelta, remainder, encodePos, encodedResult, beta,
+ blockSize, scratch.encodingType, m, scratch);
+ long subEnd = System.nanoTime();
+ if (subcolumnTime != null) {
+ subcolumnTime[0] += (subEnd - subStart);
+ }
+ return encodePos;
+ }
+
+ public static int BlockDecoder(byte[] encodedResult, int blockIndex, int blockSize,
+ int remainder, int encodePos, int[] data) {
+ int minDelta = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int base = blockIndex * blockSize;
+ encodePos = SubcolumnDecoder(encodedResult, encodePos, data, base, remainder, blockSize);
+
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] += minDelta;
+ }
+
+ return encodePos;
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, null, null);
+ }
+
+ private static void resetAlphaHybridState() {
+ EncodeScratch scratch = ENCODE_SCRATCH.get();
+ scratch.lastBestBeta = 2;
+ scratch.cachedBeta = -1;
+ scratch.cachedL = 0;
+ scratch.cachedListLength = -1;
+ }
+
+ public static int EncoderHybridAlpha(int[] data, int blockSize, byte[] encodedResult) {
+ return EncoderHybridAlpha(data, blockSize, encodedResult, null, null);
+ }
+
+ public static int EncoderHybridAlpha(int[] data, int blockSize, byte[] encodedResult,
+ long[] forTime, long[] subcolumnTime) {
+ boolean previous = USE_ALPHA_HYBRID;
+ boolean previousFast = USE_ALPHA_FAST_HYBRID;
+ USE_ALPHA_HYBRID = true;
+ USE_ALPHA_FAST_HYBRID = false;
+ resetAlphaHybridState();
+ try {
+ return Encoder(data, blockSize, encodedResult, forTime, subcolumnTime);
+ } finally {
+ USE_ALPHA_HYBRID = previous;
+ USE_ALPHA_FAST_HYBRID = previousFast;
+ }
+ }
+
+ public static int EncoderFastHybridAlpha(int[] data, int blockSize, byte[] encodedResult) {
+ return EncoderFastHybridAlpha(data, blockSize, encodedResult, null, null);
+ }
+
+ public static int EncoderFastHybridAlpha(int[] data, int blockSize, byte[] encodedResult,
+ long[] forTime, long[] subcolumnTime) {
+ boolean previous = USE_ALPHA_HYBRID;
+ boolean previousFast = USE_ALPHA_FAST_HYBRID;
+ USE_ALPHA_HYBRID = true;
+ USE_ALPHA_FAST_HYBRID = true;
+ resetAlphaHybridState();
+ try {
+ return Encoder(data, blockSize, encodedResult, forTime, subcolumnTime);
+ } finally {
+ USE_ALPHA_HYBRID = previous;
+ USE_ALPHA_FAST_HYBRID = previousFast;
+ }
+ }
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult, long[] forTime,
+ long[] subcolumnTime) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ int2Bytes(dataLength, encodePos, encodedResult);
+ encodePos += 4;
+
+ int2Bytes(blockSize, encodePos, encodedResult);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = ENCODE_SCRATCH.get().beta;
+ beta[0] = 2;
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockEncoder(data, i, blockSize, blockSize, encodePos, encodedResult,
+ beta, forTime, subcolumnTime);
+ }
+
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ int2Bytes(data[base + i], encodePos, encodedResult);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockEncoder(data, numBlocks, blockSize, remainder, encodePos,
+ encodedResult, beta, forTime, subcolumnTime);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int blockSize = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ int base = numBlocks * blockSize;
+ for (int i = 0; i < remainder; i++) {
+ data[base + i] = bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos,
+ data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ private static int[] loadCsvAsScaledInts(File file) throws IOException {
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int[] result = new int[data.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data.size(); i++) {
+ result[i] = (int) (data.get(i) * maxMul);
+ }
+ return result;
+ }
+
+ @Test
+ public void benchmarkAlphaHybrid() throws IOException {
+ String inputParentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/dataset/";
+ int blockSize = 512;
+ int warmupTime = 3;
+ int repeatTime = 30;
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ return;
+ }
+ Arrays.sort(csvFiles);
+
+ System.out.println(
+ "Dataset,Points,BaselineSize,HybridSize,BaselineRatio,HybridRatio,"
+ + "BaselineNsPerPoint,HybridNsPerPoint,TimeChangePct,SameSize");
+ for (File file : csvFiles) {
+ int[] data = loadCsvAsScaledInts(file);
+ byte[] baselineEncoded = new byte[Math.max(16, data.length * 8)];
+ byte[] hybridEncoded = new byte[Math.max(16, data.length * 8)];
+
+ int baselineLength = 0;
+ int hybridLength = 0;
+ for (int i = 0; i < warmupTime; i++) {
+ baselineLength = Encoder(data, blockSize, baselineEncoded);
+ hybridLength = EncoderHybridAlpha(data, blockSize, hybridEncoded);
+ }
+
+ long baselineTime = 0;
+ long hybridTime = 0;
+ for (int i = 0; i < repeatTime; i++) {
+ if ((i & 1) == 0) {
+ long start = System.nanoTime();
+ baselineLength = Encoder(data, blockSize, baselineEncoded);
+ baselineTime += System.nanoTime() - start;
+
+ start = System.nanoTime();
+ hybridLength = EncoderHybridAlpha(data, blockSize, hybridEncoded);
+ hybridTime += System.nanoTime() - start;
+ } else {
+ long start = System.nanoTime();
+ hybridLength = EncoderHybridAlpha(data, blockSize, hybridEncoded);
+ hybridTime += System.nanoTime() - start;
+
+ start = System.nanoTime();
+ baselineLength = Encoder(data, blockSize, baselineEncoded);
+ baselineTime += System.nanoTime() - start;
+ }
+ }
+ baselineTime /= repeatTime;
+ hybridTime /= repeatTime;
+
+ Assert.assertArrayEquals(
+ data, Decoder(Arrays.copyOf(hybridEncoded, hybridLength)));
+
+ double baselineRatio = baselineLength / (double) (Math.max(1, data.length) * Long.BYTES);
+ double hybridRatio = hybridLength / (double) (Math.max(1, data.length) * Long.BYTES);
+ double baselineNsPerPoint = baselineTime / (double) Math.max(1, data.length);
+ double hybridNsPerPoint = hybridTime / (double) Math.max(1, data.length);
+ double timeChangePct = (hybridNsPerPoint - baselineNsPerPoint)
+ / Math.max(1.0e-9, baselineNsPerPoint) * 100.0;
+
+ System.out.println(
+ extractFileName(file.toString())
+ + ","
+ + data.length
+ + ","
+ + baselineLength
+ + ","
+ + hybridLength
+ + ","
+ + baselineRatio
+ + ","
+ + hybridRatio
+ + ","
+ + baselineNsPerPoint
+ + ","
+ + hybridNsPerPoint
+ + ","
+ + timeChangePct
+ + ","
+ + (baselineLength == hybridLength));
+ }
+ }
+
+ @Test
+ public void benchmarkFastAlphaHybrid() throws IOException {
+ String inputParentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/dataset/";
+ int blockSize = 512;
+ int warmupTime = 3;
+ int repeatTime = 30;
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ return;
+ }
+ Arrays.sort(csvFiles);
+
+ System.out.println(
+ "Dataset,Points,HybridSize,FastSize,HybridRatio,FastRatio,"
+ + "HybridNsPerPoint,FastNsPerPoint,FastVsHybridTimePct,"
+ + "FastVsHybridSizeDelta,FastNoWorseRatio,FastFaster");
+ for (File file : csvFiles) {
+ int[] data = loadCsvAsScaledInts(file);
+ byte[] hybridEncoded = new byte[Math.max(16, data.length * 8)];
+ byte[] fastEncoded = new byte[Math.max(16, data.length * 8)];
+
+ int hybridLength = 0;
+ int fastLength = 0;
+ for (int i = 0; i < warmupTime; i++) {
+ hybridLength = EncoderHybridAlpha(data, blockSize, hybridEncoded);
+ fastLength = EncoderFastHybridAlpha(data, blockSize, fastEncoded);
+ }
+
+ long start = System.nanoTime();
+ for (int i = 0; i < repeatTime; i++) {
+ hybridLength = EncoderHybridAlpha(data, blockSize, hybridEncoded);
+ }
+ long hybridTime = (System.nanoTime() - start) / repeatTime;
+
+ start = System.nanoTime();
+ for (int i = 0; i < repeatTime; i++) {
+ fastLength = EncoderFastHybridAlpha(data, blockSize, fastEncoded);
+ }
+ long fastTime = (System.nanoTime() - start) / repeatTime;
+
+ Assert.assertArrayEquals(data, Decoder(Arrays.copyOf(fastEncoded, fastLength)));
+
+ double hybridRatio = hybridLength / (double) (Math.max(1, data.length) * Long.BYTES);
+ double fastRatio = fastLength / (double) (Math.max(1, data.length) * Long.BYTES);
+ double hybridNsPerPoint = hybridTime / (double) Math.max(1, data.length);
+ double fastNsPerPoint = fastTime / (double) Math.max(1, data.length);
+ double timeChangePct = (fastNsPerPoint - hybridNsPerPoint)
+ / Math.max(1.0e-9, hybridNsPerPoint) * 100.0;
+
+ System.out.println(
+ extractFileName(file.toString())
+ + ","
+ + data.length
+ + ","
+ + hybridLength
+ + ","
+ + fastLength
+ + ","
+ + hybridRatio
+ + ","
+ + fastRatio
+ + ","
+ + hybridNsPerPoint
+ + ","
+ + fastNsPerPoint
+ + ","
+ + timeChangePct
+ + ","
+ + (fastLength - hybridLength)
+ + ","
+ + (fastLength <= hybridLength)
+ + ","
+ + (fastNsPerPoint < hybridNsPerPoint));
+ }
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "subcolumn_adddict_prunenew_opt2.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ "For Time",
+ "Subcolumn Encode Time"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, data2Arr.length * 8)];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ long forTime = 0;
+ long subcolumnEncodeTime = 0;
+ double compressedSize = 0;
+ int length = 0;
+ long[] forTimeArr = new long[1];
+ long[] subcolumnEncodeTimeArr = new long[1];
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ forTimeArr[0] = 0;
+ subcolumnEncodeTimeArr[0] = 0;
+ length = Encoder(data2Arr, blockSize, encodedResult, forTimeArr,
+ subcolumnEncodeTimeArr);
+ forTime += forTimeArr[0];
+ subcolumnEncodeTime += subcolumnEncodeTimeArr[0];
+ }
+ long e = System.nanoTime();
+ encodeTime += (e - s) / repeatTime;
+ forTime /= repeatTime;
+ subcolumnEncodeTime /= repeatTime;
+ compressedSize += length;
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s) / repeatTime;
+
+ double ratio = compressedSize / (double) (Math.max(1, data1.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "Sub-columns(AddDictPruneNew-Opt2)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio),
+ String.valueOf(forTime),
+ String.valueOf(subcolumnEncodeTime)
+ });
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ private static int[] syntheticData(int size, int seed) {
+ int[] data = new int[size];
+ int state = seed;
+ for (int i = 0; i < size; i++) {
+ state = state * 1103515245 + 12345;
+ data[i] = (state >>> 16) & 0x7FFF;
+ if (i % 17 == 0) {
+ data[i] = data[Math.max(0, i - 1)];
+ }
+ }
+ return data;
+ }
+
+ @Test
+ public void testRoundTripAfterOpt3() {
+ int blockSize = 512;
+ int[][] patterns = {
+ syntheticData(2048, 1),
+ syntheticData(4096, 7),
+ syntheticData(8192, 42)
+ };
+ byte[] encoded = new byte[65536];
+ for (int[] data : patterns) {
+ int len = Encoder(data, blockSize, encoded);
+ int[] decoded = Decoder(Arrays.copyOf(encoded, len));
+ Assert.assertArrayEquals(data, decoded);
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneTest.java
new file mode 100644
index 0000000..c5e2b26
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnPruneTest.java
@@ -0,0 +1,997 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+public class SubcolumnPruneTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size, int[] encodingType) {
+
+ if (m == 0) {
+ return 1;
+ }
+
+ int betaBest = 1;
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int[] threshold = null;
+
+ switch(block_size) {
+ case 32:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ case 64:
+ threshold = new int[] {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ break;
+ case 128:
+ threshold = new int[] {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ break;
+ case 256:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ break;
+ case 512:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ break;
+ case 1024:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ break;
+ case 2048:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ break;
+ case 4096:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ break;
+ case 8192:
+ threshold = new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ break;
+ default:
+ threshold = new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ break;
+ }
+
+ int cost1 = 0;
+
+ // System.out.println("x:");
+ // for (int i = 0; i < x_length; i++) {
+ // System.out.print(x[i] + " ");
+ // }
+ // System.out.println();
+
+ BitSet[] bitsets = new BitSet[m];
+
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+
+ for (int i = 0; i < m; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int current_value = (x[0] >> i) & 1;
+
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ int count = 0;
+
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+
+ for (int j = 1; j < x_length; j++) {
+
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+
+ bitsets[i].set(j - 1);
+ }
+
+ }
+
+ bitsets[i].set(x_length - 1);
+
+ count++;
+
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+
+ if (bpe_cost_single[i] <= rle_cost_single[i] && bpe_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 0; // bpe
+ cost1 += bpe_cost_single[i];
+ } else if (rle_cost_single[i] < bpe_cost_single[i] && rle_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 1; // rle
+ cost1 += rle_cost_single[i];
+ } else {
+ encodingType[i] = 2; // de
+ cost1 += de_cost_single[i];
+ }
+
+ }
+
+ int cMin = cost1;
+
+ // int[] beta_list = new int[m - 1];
+ // for (int i = 0; i < m - 1; i++) {
+ // beta_list[i] = i + 2;
+ // }
+
+ int[] beta_list = { 2, 3, 4 };
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int cost = 0;
+
+ int[] encodingTypeTemp = new int[l];
+
+ for (int i = 0; i < l; i++) {
+ // System.out.println("subcolumn index: " + i);
+
+ int currentCost = 0;
+
+ int bpCost = 0;
+
+ int beta_start = (Math.min(m - 1, (i + 1) * beta - 1));
+ while (beta_start >= i * beta && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+
+ if (beta_start < i * beta) {
+ beta_start = i * beta;
+ }
+
+ bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+
+ // System.out.println("bpCost: " + bpCost);
+
+ currentCost = bpCost;
+
+ int rleCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+
+ if (rleCostMax < currentCost) {
+ // if (rle_cost_single[i * beta] < currentCost) {
+ int rleCost = 0;
+
+ boolean currentBetter = false;
+
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (currentCost > rleCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+
+ }
+
+ int deCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (de_cost_single[j] > deCostMax) {
+ deCostMax = de_cost_single[j];
+ }
+ }
+
+ if (deCostMax < currentCost) {
+ // if (de_cost_single[i * beta] < currentCost) {
+ boolean currentBetter = false;
+ Set<Integer> uniqueValues = new HashSet<>();
+
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ currentBetter = true;
+ break;
+ }
+ }
+
+ if (!currentBetter) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+
+ if (deCost < currentCost) {
+ currentCost = deCost;
+
+ encodingTypeTemp[i] = 2;
+ }
+ }
+
+ }
+
+ cost += currentCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+
+ // System.out.println("betaBest: " + betaBest);
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size, int[] encodingType) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ // System.out.println("maxValue: " + maxValue);
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ // System.out.println("All zero list.");
+ return encode_pos;
+ }
+
+ int l;
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+
+ if (encodingType[i] == 2) {
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ int dict_bit_width = bitWidth(cardinality) ;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ }
+
+ index++;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if(type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ encode_pos =decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ int[] encodingType = new int[m];
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size, encodingType);
+
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size, encodingType);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_dictionary.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Dictionary)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount2Test.java
new file mode 100644
index 0000000..e180fdb
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount2Test.java
@@ -0,0 +1,135 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryCount2Test {
+
+ public static void Query(byte[] encoded_result) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuery(encoded_result, i, block_size,
+ block_size, encode_pos,
+ result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ int value = SubcolumnTest.bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ result[result_length[0]]++;
+ }
+ } else {
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size,
+ remainder, encode_pos,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ // int[] block_data = new int[remainder];
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ result[result_length[0]] += remainder;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ // int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ // encode_pos = (encode_pos * 8 + bw * index + 7) / 8;
+ // encode_pos = (encode_pos * 8 + bitWidthList[i] * index + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ }
+ }
+
+ // if (target <= 0) {
+ // for (int i = 0; i < remainder; i++) {
+ // result[result_length[0]] = block_size * block_index + i;
+ // result_length[0]++;
+ // }
+ // return encode_pos;
+ // }
+
+ result[result_length[0]] += remainder;
+
+ return encode_pos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount3Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount3Test.java
new file mode 100644
index 0000000..9f7ef36
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCount3Test.java
@@ -0,0 +1,41 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryCount3Test {
+
+ public static void Query(byte[] encoded_result) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+
+ result[0] = data_length;
+
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountNewTest.java
new file mode 100644
index 0000000..b794c72
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountNewTest.java
@@ -0,0 +1,176 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryCountNewTest {
+
+ public static void Query(byte[] encoded_result, int target) {
+ int encodePos = 0;
+
+ // 解析数据长度
+ int data_length = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ // 解析块大小
+ int block_size = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[1]; // 只记录计数
+ result[0] = 0;
+
+ // 处理每个块
+ for (int i = 0; i < num_blocks; i++) {
+ encodePos = BlockQueryCount(encoded_result, i, block_size,
+ block_size, encodePos, target, result);
+ }
+
+ // 处理剩余部分
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = SubcolumnTest.bytes2Integer(encoded_result, encodePos, 4);
+ encodePos += 4;
+ if (value == target) {
+ result[0]++;
+ }
+ }
+ } else {
+ encodePos = BlockQueryCount(encoded_result, num_blocks, block_size,
+ remainder, encodePos, target, result);
+ }
+ }
+
+ public static int BlockQueryCount(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encodePos, int target, int[] result) {
+
+ // 读取最小增量
+ int minDelta0 = SubcolumnTest.bytes2Integer(encoded_result, encodePos, 4);
+ encodePos += 4;
+
+ int m = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ target -= minDelta0; // 调整目标值
+
+ // 初始化候选索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = remainder;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ result[0] += remainder;
+ }
+ return encodePos;
+ }
+
+ int bitWidth = SubcolumnTest.bitWidth(block_size);
+ int beta = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ // 计算子列数
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 1, l, encodingType);
+
+ // 处理每个子列
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) { // 类型 0：plain bit-packed
+ if (target < 0) {
+ long bitPos = ((long) encodePos) * 8L + (long) bitWidthList[i] * (long) remainder;
+ encodePos = (int) ((bitPos + 7) / 8);
+ continue;
+ }
+
+ long bitPos = ((long) encodePos) * 8L;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+ int current = SubcolumnTest.bytesToInt(encoded_result,
+ (int)(bitPos + index * bitWidthList[i]), bitWidthList[i]);
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+ if (current == value) {
+ candidate_indices[new_length++] = index;
+ }
+ }
+
+ candidate_length = new_length;
+ bitPos += (long) remainder * bitWidthList[i];
+ encodePos = (int) ((bitPos + 7) / 8);
+
+ } else { // 类型 1：RLE + bitpacked values
+ int index = ((encoded_result[encodePos] & 0xFF) << 8) | (encoded_result[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ if (target < 0) {
+ long bitPos = ((long) encodePos) * 8L + (long) bitWidth * index;
+ encodePos = (int) ((bitPos + 7) / 8);
+ bitPos = ((long) encodePos) * 8L + (long) bitWidthList[i] * index;
+ encodePos = (int) ((bitPos + 7) / 8);
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidth, index, run_length);
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidthList[i], index, rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index && rle_values[rleIndex] == value) {
+ candidate_indices[new_length++] = index_candidate;
+ }
+ }
+
+ candidate_length = new_length;
+ }
+ }
+
+ result[0] += candidate_length; // 统计符合条件的总数
+
+ return encodePos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountTest.java
index b4f6b13..dc076de 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryCountTest.java
@@ -45,13 +45,14 @@
if (remainder <= 3) {
for (int i = 0; i < remainder; i++) {
- int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
- ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ int value = SubcolumnTest.bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
if (value == target) {
result[result_length[0]]++;
}
- encode_pos += 4;
}
} else {
encode_pos = BlockQueryCount(encoded_result, num_blocks, block_size,
@@ -216,319 +217,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_count_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- ratio += ratioTmp;
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_count_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- ratio += ratioTmp;
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualNewTest.java
new file mode 100644
index 0000000..1fdba68
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualNewTest.java
@@ -0,0 +1,214 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryEqualNewTest {
+
+ public static void Query(byte[] encoded_result, int target) {
+ int encodePos = 0;
+
+ // 解析数据长度
+ int data_length = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ // 解析块大小
+ int block_size = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ // 处理每个块
+ for (int i = 0; i < num_blocks; i++) {
+ encodePos = BlockQueryIndex(encoded_result, i, block_size,
+ block_size, encodePos, target,
+ result, result_length);
+ }
+
+ // 处理剩余部分
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = SubcolumnTest.bytes2Integer(encoded_result, encodePos, 4);
+ encodePos += 4;
+ if (value == target) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ }
+ } else {
+ encodePos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encodePos, target, result, result_length);
+ }
+ }
+
+
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encodePos, int target, int[] result, int[] result_length) {
+
+ // 读取最小增量
+ int minDelta0 = SubcolumnTest.bytes2Integer(encoded_result, encodePos, 4);
+ encodePos += 4;
+
+ int m = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ target -= minDelta0; // 调整目标值
+
+ if (m == 0) {
+ if (target == 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]++] = block_size * block_index + i;
+ }
+ }
+ return encodePos;
+ }
+
+ // 初始化候选索引（一次分配）
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = remainder;
+ for (int i = 0; i < remainder; i++) candidate_indices[i] = i;
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+ int beta = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 1, l, encodingType);
+
+ // 处理每个子列（从高位到低位）
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+
+ if (type == 0) { // plain bit-packed
+ if (target < 0) {
+ // skip this subcolumn: compute byte position advance
+ long skipBits = (long) bitWidth * (long) remainder;
+ long bitPos = ((long) encodePos) * 8L + skipBits;
+ encodePos = (int) ((bitPos + 7L) >>> 3);
+ continue;
+ }
+
+ final int expectedValue = (target >> (i * beta)) & ((1 << beta) - 1);
+ long baseBitPos = ((long) encodePos) * 8L; // start bit pos of this subcolumn
+
+ // Heuristic: if many candidates remain, sequential scan is better;
+ // otherwise random-access per candidate is better.
+ if (candidate_length > (remainder >> 1)) {
+ // sequential scan across all remainder values
+ int new_length = 0;
+ long bitPos = baseBitPos;
+ for (int pos = 0; pos < remainder; pos++) {
+ int subValue = SubcolumnTest.bytesToInt(encoded_result, (int) (bitPos + (long) pos * bitWidth),
+ bitWidth);
+ if (subValue == expectedValue) {
+ candidate_indices[new_length++] = pos;
+ }
+ }
+ candidate_length = new_length;
+ } else {
+ // random-access for just the candidate positions (current approach),
+ // but avoid recomputing expectedValue and some arithmetic.
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int idx = candidate_indices[j];
+ int subValue = SubcolumnTest.bytesToInt(encoded_result,
+ (int) (baseBitPos + (long) idx * bitWidth), bitWidth);
+ if (subValue == expectedValue) {
+ candidate_indices[new_length++] = idx;
+ }
+ }
+ candidate_length = new_length;
+ }
+
+ // advance encodePos past this packed block
+ long advBits = (long) bitWidth * (long) remainder;
+ long endBitPos = baseBitPos + advBits;
+ encodePos = (int) ((endBitPos + 7L) >>> 3);
+
+ } else { // type == 1: RLE + bitpacked values
+ // read length of runs (index)
+ int index = ((encoded_result[encodePos] & 0xFF) << 8) | (encoded_result[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ if (target < 0) {
+ // skip both run_length and rle_values payloads:
+ long skipBitsRunLens = (long) bw * index;
+ long bitPos = ((long) encodePos) * 8L + skipBitsRunLens;
+ encodePos = (int) ((bitPos + 7L) >>> 3);
+
+ long skipBitsValues = (long) bitWidth * index;
+ bitPos = ((long) encodePos) * 8L + skipBitsValues;
+ encodePos = (int) ((bitPos + 7L) >>> 3);
+ continue;
+ }
+
+ // decode run lengths and values
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bw, index, run_length);
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidth, index, rle_values);
+
+ final int expectedValue = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ // Candidate indices are sorted; advance rleIndex monotonically.
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0; // current starting pos of the run rleIndex
+
+ for (int j = 0; j < candidate_length; j++) {
+ int idxCandidate = candidate_indices[j];
+
+ // move rleIndex forward until its run covers idxCandidate (or we run out)
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= idxCandidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index && rle_values[rleIndex] == expectedValue) {
+ // idxCandidate falls into a run with matching value
+ candidate_indices[new_length++] = idxCandidate;
+ }
+ }
+
+ candidate_length = new_length;
+ }
+ }
+
+ // 输出最终匹配的索引
+ for (int i = 0; i < candidate_length; i++) {
+ result[result_length[0]++] = block_size * block_index + candidate_indices[i];
+ }
+
+ return encodePos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualTest.java
index 1272bc2..33c23ac 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryEqualTest.java
@@ -45,14 +45,15 @@
if (remainder <= 3) {
for (int i = 0; i < remainder; i++) {
- int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
- ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ // int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ // ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ int value = SubcolumnTest.bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
if (value == target) {
result[result_length[0]] = value;
result_length[0]++;
}
- encode_pos += 4;
}
} else {
encode_pos = BlockQueryIndex(encoded_result, num_blocks, block_size,
@@ -66,8 +67,9 @@
int encode_pos, int target, int[] result, int[] result_length) {
int[] min_delta = new int[3];
- min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
- ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ // min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ // ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ min_delta[0] = SubcolumnTest.bytes2Integer(encoded_result, encode_pos, 4);
encode_pos += 4;
// int[] block_data = new int[remainder];
@@ -221,314 +223,4 @@
return encode_pos;
}
-
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_equal_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_equal_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterLessTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterLessTest.java
index 48bb58a..8ee26fc 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterLessTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterLessTest.java
@@ -202,346 +202,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryGreaterRange = new HashMap<>();
-
- queryGreaterRange.put("Bird-migration", 2500000);
- queryGreaterRange.put("Bitcoin-price", 160000000);
- queryGreaterRange.put("City-temp", 480);
- queryGreaterRange.put("Dewpoint-temp", 9500);
- queryGreaterRange.put("IR-bio-temp", -300);
- queryGreaterRange.put("PM10-dust", 1000);
- queryGreaterRange.put("Stocks-DE", 40000);
- queryGreaterRange.put("Stocks-UK", 20000);
- queryGreaterRange.put("Stocks-USA", 5000);
- queryGreaterRange.put("Wind-Speed", 50);
- queryGreaterRange.put("Wine-Tasting", 0);
-
- HashMap<String, Integer> queryLessRange = new HashMap<>();
-
- queryLessRange.put("Bird-migration", 2600000);
- queryLessRange.put("Bitcoin-price", 170000000);
- queryLessRange.put("City-temp", 700);
- queryLessRange.put("Dewpoint-temp", 9600);
- queryLessRange.put("IR-bio-temp", -200);
- queryLessRange.put("PM10-dust", 2000);
- queryLessRange.put("Stocks-DE", 90000);
- queryLessRange.put("Stocks-UK", 30000);
- queryLessRange.put("Stocks-USA", 6000);
- queryLessRange.put("Wind-Speed", 60);
- queryLessRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_greater_less_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryGreaterRange.get(datasetName),
- queryLessRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryGreaterRange = new HashMap<>();
-
- queryGreaterRange.put("Bird-migration", 2500000);
- queryGreaterRange.put("Bitcoin-price", 160000000);
- queryGreaterRange.put("City-temp", 480);
- queryGreaterRange.put("Dewpoint-temp", 9500);
- queryGreaterRange.put("IR-bio-temp", -300);
- queryGreaterRange.put("PM10-dust", 1000);
- queryGreaterRange.put("Stocks-DE", 40000);
- queryGreaterRange.put("Stocks-UK", 20000);
- queryGreaterRange.put("Stocks-USA", 5000);
- queryGreaterRange.put("Wind-Speed", 50);
- queryGreaterRange.put("Wine-Tasting", 0);
-
- HashMap<String, Integer> queryLessRange = new HashMap<>();
-
- queryLessRange.put("Bird-migration", 2600000);
- queryLessRange.put("Bitcoin-price", 170000000);
- queryLessRange.put("City-temp", 700);
- queryLessRange.put("Dewpoint-temp", 9600);
- queryLessRange.put("IR-bio-temp", -200);
- queryLessRange.put("PM10-dust", 2000);
- queryLessRange.put("Stocks-DE", 90000);
- queryLessRange.put("Stocks-UK", 30000);
- queryLessRange.put("Stocks-USA", 6000);
- queryLessRange.put("Wind-Speed", 60);
- queryLessRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_greater_less_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryGreaterLessTest.Query(encoded_result,
- queryGreaterRange.get(datasetName),
- queryLessRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterNewTest.java
new file mode 100644
index 0000000..b6bbb6a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterNewTest.java
@@ -0,0 +1,220 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryGreaterNewTest {
+ public static void Query(byte[] encoded_result, int lower_bound) {
+ int encodePos = 0;
+
+ int data_length = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int block_size = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encodePos = BlockQueryIndex(encoded_result, i, block_size, block_size,
+ encodePos, lower_bound, result, result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ if (value > lower_bound) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encodePos, lower_bound, result, result_length);
+ }
+
+ // 可选：返回结果或者把 result/result_length 放到可访问位置
+ }
+
+ /**
+ * 返回更新后的字节偏移 encodePos（保持和原来接口一致）。
+ *
+ * 注意：
+ * - encodePos 传入/返回的是字节偏移（byte offset）。
+ * - 当需要按位跳过数据时，使用临时 long bitPos = encodePos * 8L 来处理，再换算回字节偏移。
+ */
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encodePos, int lower_bound, int[] result, int[] result_length) {
+
+ // 只用第一个 min_delta（原代码也只用了第一个）
+ int minDelta0 = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int m = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int adjustedLower = lower_bound - minDelta0; // 不更改传入 lower_bound 的原始值
+
+ // 初始化候选索引（全部候选）
+ int[] candidate_indices = new int[Math.max(1, remainder)];
+ int candidate_length = remainder;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ }
+
+ if (m == 0) {
+ if (adjustedLower < 0) {
+ int baseIndex = block_size * block_index;
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = baseIndex + i;
+ result_length[0]++;
+ }
+ }
+ return encodePos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+ int beta = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 1, l, encodingType);
+
+ int baseIndex = block_size * block_index;
+
+ // 处理每个子列（从高到低，与原代码一致）
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+ // 类型 0：plain bit-packed
+ if (adjustedLower <= 0) {
+ // 只跳过该子列的所有位宽 -> 使用位偏移处理避免破坏 encodePos 的语义
+ long bitPos = ((long) encodePos) * 8L + (long) bitWidthList[i] * (long) remainder;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+ continue;
+ }
+
+ // 需要解码并检查候选
+ long bitPos = ((long) encodePos) * 8L;
+ int new_length = 0;
+
+ // 每个候选索引读取该子列对应的 bit-width 值并比较
+ int shiftMaskValue = (adjustedLower >> (i * beta)) & ((1 << beta) - 1);
+ int bw_i = bitWidthList[i];
+
+ for (int j = 0; j < candidate_length; j++) {
+ int idx = candidate_indices[j];
+ // 从 bitPos + idx*bw_i 处解出值（假设 bytesToInt 的第二个参数表示 bit-offset）
+ int bitOffsetForThis = (int) (bitPos + (long) idx * bw_i);
+ int subValue = SubcolumnTest.bytesToInt(encoded_result, bitOffsetForThis, bw_i);
+ if (subValue > shiftMaskValue) {
+ result[result_length[0]] = baseIndex + idx;
+ result_length[0]++;
+ } else if (subValue == shiftMaskValue) {
+ candidate_indices[new_length++] = idx;
+ }
+ }
+
+ candidate_length = new_length;
+ // advance encodePos by remainder * bw_i bits
+ bitPos += (long) remainder * bw_i;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+
+ } else {
+ // type == 1：RLE + bitpacked values
+ // 先读 index（RLE segment count）
+ int index = ((encoded_result[encodePos] & 0xFF) << 8) | (encoded_result[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ if (adjustedLower <= 0) {
+ // 跳过 RLE 的两个区域（先按 bw 跳过 run_length，再按 bitWidthList 跳过 rle_values）
+ long bitPos = ((long) encodePos) * 8L;
+ bitPos += (long) bw * index;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+
+ bitPos = ((long) encodePos) * 8L;
+ bitPos += (long) bitWidthList[i] * index;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+ continue;
+ }
+
+ // 读取 run lengths（bw 位宽）和对应的 rle_values（bitWidthList[i] 位宽）
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bw, index, run_length);
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidthList[i], index,
+ rle_values);
+
+ // 遍历候选索引并用 RLE 查找对应的 value
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int targetValue = (adjustedLower >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int idx = candidate_indices[j];
+ // 移动到包含 idx 的 rle 段
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= idx) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+ if (rleIndex < index) {
+ int rv = rle_values[rleIndex];
+ if (rv > targetValue) {
+ result[result_length[0]] = baseIndex + idx;
+ result_length[0]++;
+ } else if (rv == targetValue) {
+ candidate_indices[new_length++] = idx;
+ }
+ }
+ }
+ candidate_length = new_length;
+ }
+ }
+
+ // 遍历所有子列后，如果 adjustedLower <= 0，则整块全部命中（和原逻辑一致）
+ if (adjustedLower <= 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = baseIndex + i;
+ result_length[0]++;
+ }
+ }
+
+ return encodePos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterTest.java
index 0fbbecf..7848eec 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryGreaterTest.java
@@ -16,6 +16,13 @@
public class SubcolumnQueryGreaterTest {
+ // 在类里添加一个全局可复用候选缓冲（避免每块 new）
+ private static final ThreadLocal<int[]> THREAD_CAND_BUF = ThreadLocal.withInitial(() -> new int[8192]); // 初始大小按最大 block_size 调整
+ private static final ThreadLocal<int[]> THREAD_RUN_BUF = ThreadLocal.withInitial(() -> new int[1024]);
+ private static final ThreadLocal<int[]> THREAD_VAL_BUF = ThreadLocal.withInitial(() -> new int[1024]);
+
+ // 更快的 readBits（放到类中）
+
public static void Query(byte[] encoded_result, int lower_bound) {
int encode_pos = 0;
@@ -210,313 +217,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2500000);
- queryRange.put("Bitcoin-price", 160000000);
- queryRange.put("City-temp", 480);
- queryRange.put("Dewpoint-temp", 9500);
- queryRange.put("IR-bio-temp", -300);
- queryRange.put("PM10-dust", 1000);
- queryRange.put("Stocks-DE", 40000);
- queryRange.put("Stocks-UK", 20000);
- queryRange.put("Stocks-USA", 5000);
- queryRange.put("Wind-Speed", 50);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_greater_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2500000);
- queryRange.put("Bitcoin-price", 160000000);
- queryRange.put("City-temp", 480);
- queryRange.put("Dewpoint-temp", 9500);
- queryRange.put("IR-bio-temp", -300);
- queryRange.put("PM10-dust", 1000);
- queryRange.put("Stocks-DE", 40000);
- queryRange.put("Stocks-UK", 20000);
- queryRange.put("Stocks-USA", 5000);
- queryRange.put("Wind-Speed", 50);
- queryRange.put("Wine-Tasting", 0);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_greater_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessNewTest.java
new file mode 100644
index 0000000..4628e9f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessNewTest.java
@@ -0,0 +1,200 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryLessNewTest {
+
+ public static void Query(byte[] encoded_result, int upper_bound) {
+ int encodePos = 0;
+
+ // 解析数据长度
+ int data_length = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ // 解析块大小
+ int block_size = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ // 处理每个块
+ for (int i = 0; i < num_blocks; i++) {
+ encodePos = BlockQueryIndex(encoded_result, i, block_size, block_size,
+ encodePos, upper_bound, result, result_length);
+ }
+
+ // 处理剩余部分
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockQueryIndex(encoded_result, num_blocks, block_size,
+ remainder, encodePos, upper_bound, result, result_length);
+ }
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encodePos, int upper_bound, int[] result, int[] result_length) {
+
+ // 读取最小增量
+ int minDelta0 = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int m = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int adjustedUpper = upper_bound - minDelta0; // 不更改传入的 upper_bound 原始值
+
+ // 初始化候选索引
+ int[] candidate_indices = new int[Math.max(1, remainder)];
+ int candidate_length = remainder;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ }
+
+ if (m == 0) {
+ if (adjustedUpper > 0) {
+ int baseIndex = block_size * block_index;
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = baseIndex + i;
+ result_length[0]++;
+ }
+ }
+ return encodePos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+ int beta = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 1, l, encodingType);
+
+ int baseIndex = block_size * block_index;
+
+ // 处理每个子列
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+ // 处理类型 0：plain bit-packed
+ if (adjustedUpper <= 0) {
+ long bitPos = ((long) encodePos) * 8L + (long) bitWidthList[i] * (long) remainder;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+ continue;
+ }
+
+ // 需要解码并检查候选
+ long bitPos = ((long) encodePos) * 8L;
+ int new_length = 0;
+
+ int shiftMaskValue = (adjustedUpper >> (i * beta)) & ((1 << beta) - 1);
+ int bw_i = bitWidthList[i];
+
+ for (int j = 0; j < candidate_length; j++) {
+ int idx = candidate_indices[j];
+ int bitOffsetForThis = (int) (bitPos + (long) idx * bw_i);
+ int subValue = SubcolumnTest.bytesToInt(encoded_result, bitOffsetForThis, bw_i);
+ if (subValue < shiftMaskValue) {
+ result[result_length[0]] = baseIndex + idx;
+ result_length[0]++;
+ } else if (subValue == shiftMaskValue) {
+ candidate_indices[new_length++] = idx;
+ }
+ }
+
+ candidate_length = new_length;
+ bitPos += (long) remainder * bw_i;
+ encodePos = (int) ((bitPos + 7L) / 8L);
+ } else {
+ // 处理类型 1：RLE + bitpacked values
+ int index = ((encoded_result[encodePos] & 0xFF) << 8) | (encoded_result[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ if (adjustedUpper <= 0) {
+ encodePos *= 8;
+ encodePos += bw * index;
+ encodePos = (encodePos + 7) / 8;
+
+ encodePos *= 8;
+ encodePos += bitWidthList[i] * index;
+ encodePos = (encodePos + 7) / 8;
+ continue;
+ }
+
+ // 读取 run lengths 和 rle values
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bw, index, run_length);
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidthList[i], index, rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (adjustedUpper >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = baseIndex + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length++] = index_candidate;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+ }
+ }
+
+ return encodePos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsNewTest.java
new file mode 100644
index 0000000..0938608
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsNewTest.java
@@ -0,0 +1,461 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryLessPartsNewTest {
+
+ public static void QueryTwoColumns(byte[] encoded_result1, byte[] encoded_result2, int upper_bound1,
+ int upper_bound2) {
+ int[] firstColumnResults = new int[encoded_result1.length]; // 用驼峰命名法提高可读性
+ int[] firstResultLength = new int[1];
+
+ Query(encoded_result1, upper_bound1, firstColumnResults, firstResultLength);
+
+ int[] finalResults = new int[firstResultLength[0]];
+ int[] finalResultLength = new int[1];
+
+ QueryWithIndices(encoded_result2, upper_bound2, firstColumnResults, firstResultLength[0],
+ finalResults, finalResultLength);
+ }
+
+ public static void QueryWithIndices(byte[] encoded_result, int upper_bound,
+ int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int encodePos = 0; // 变量名更新为驼峰命名法
+
+ int dataLength = ((encoded_result[encodePos] & 0xFF) << 24) | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8) | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int blockSize = ((encoded_result[encodePos] & 0xFF) << 24) | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8) | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+
+ // 初始化结果索引
+ result_length[0] = 0;
+
+ int[] blockIndicesCount = new int[numBlocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / blockSize;
+ blockIndicesCount[blockIndex]++;
+ }
+
+ int[][] blockIndices = new int[numBlocks + 1][];
+ for (int i = 0; i <= numBlocks; i++) {
+ blockIndices[i] = new int[blockIndicesCount[i]];
+ }
+
+ int[] currentIndices = new int[numBlocks + 1];
+
+ for (int i = 0; i < candidate_length; i++) {
+ int index = candidate_indices[i];
+ int blockIndex = index / blockSize;
+ int localIndex = index % blockSize;
+
+ blockIndices[blockIndex][currentIndices[blockIndex]] = localIndex;
+ currentIndices[blockIndex]++;
+ }
+
+ // 遍历所有块
+ for (int i = 0; i < numBlocks; i++) {
+ if (blockIndicesCount[i] == 0) {
+ // 计算跳过此块所需的字节数
+ encodePos = SkipBlock(encoded_result, i, blockSize, blockSize, encodePos);
+ continue;
+ }
+
+ // 对该块中的候选索引执行查询
+ encodePos = BlockQueryWithIndices(encoded_result, i, blockSize,
+ blockSize, encodePos, upper_bound,
+ blockIndices[i], blockIndicesCount[i], result, result_length);
+ }
+
+ int remainder = dataLength % blockSize;
+
+ if (remainder > 0) {
+ if (blockIndicesCount[numBlocks] > 0) {
+ if (remainder <= 3) {
+ for (int j = 0; j < blockIndicesCount[numBlocks]; j++) {
+ int idx = blockIndices[numBlocks][j];
+ int offset = numBlocks * blockSize + idx;
+ if (offset < dataLength) {
+ int value = ((encoded_result[encodePos + idx * 4] & 0xFF) << 24)
+ | ((encoded_result[encodePos + idx * 4 + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + idx * 4 + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + idx * 4 + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = offset;
+ result_length[0]++;
+ }
+ }
+ }
+ encodePos += remainder * 4;
+ } else {
+ encodePos = BlockQueryWithIndices(encoded_result, numBlocks, blockSize,
+ remainder, encodePos, upper_bound,
+ blockIndices[numBlocks], blockIndicesCount[numBlocks], result, result_length);
+ }
+ } else {
+ // 没有候选索引，跳过剩余部分
+ encodePos += (remainder <= 3) ? remainder * 4 : SkipBlock(encoded_result, numBlocks, blockSize, remainder, encodePos);
+ }
+ }
+ }
+
+ public static int BlockQueryWithIndices(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] candidate_indices, int candidate_length,
+ int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ | ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 所有索引默认都是候选索引
+ int[] filtered_indices = new int[candidate_length];
+ int filtered_length = candidate_length;
+ System.arraycopy(candidate_indices, 0, filtered_indices, 0, candidate_length);
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < filtered_length; i++) {
+ result[result_length[0]] = block_size * block_index + filtered_indices[i];
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8);
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < filtered_length; j++) {
+ int index = filtered_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ filtered_indices[new_length++] = index; // 简化赋值逻辑
+ }
+ }
+
+ filtered_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8; // 计算偏移并清晰表明目的
+
+ } else {
+ int rleIndex = 0; // 为每个候选索引查找对应的RLE值
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos = (int) (((long) encode_pos * 8 + bw * index + 7) / 8);
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * index + 7) / 8);
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ // 为每个候选索引查找对应的RLE值
+ for (int j = 0; j < filtered_length; j++) {
+ int index_candidate = filtered_indices[j];
+
+ // 查找包含此索引的RLE段
+ int currentPos = 0;
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ filtered_indices[new_length++] = index_candidate; // 简化赋值逻辑
+ }
+ }
+ }
+
+ filtered_length = new_length;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ private static int SkipBlock(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos) {
+ encode_pos += 4; // 计算 min_delta 时直接跳过不需要使用
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+
+ if (type == 0) {
+ // 类型 0，直接跳过
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+
+ encode_pos = (int) (((long) encode_pos * 8 + bw * index + 7) / 8);
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * index + 7) / 8);
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static void Query(byte[] encoded_result, int upper_bound, int[] result, int[] result_length) {
+ int encodePos = 0; // 更新为驼峰命名法
+
+ int dataLength = ((encoded_result[encodePos] & 0xFF) << 24) | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8) | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int blockSize = ((encoded_result[encodePos] & 0xFF) << 24) | ((encoded_result[encodePos + 1] & 0xFF) << 16) |
+ ((encoded_result[encodePos + 2] & 0xFF) << 8) | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+
+ // 查询结果
+ result_length[0] = 0;
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockQueryIndex(encoded_result, i, blockSize,
+ blockSize, encodePos, upper_bound,
+ result, result_length);
+ }
+
+ int remainder = dataLength % blockSize;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encodePos] & 0xFF) << 24) |
+ ((encoded_result[encodePos + 1] & 0xFF) << 16) |
+ ((encoded_result[encodePos + 2] & 0xFF) << 8) | (encoded_result[encodePos + 3] & 0xFF);
+ if (value < upper_bound) {
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockQueryIndex(encoded_result, numBlocks, blockSize,
+ remainder, encodePos, upper_bound,
+ result, result_length);
+ }
+
+ }
+
+ public static int BlockQueryIndex(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int upper_bound, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ upper_bound -= min_delta[0];
+
+ // 候选索引列表，当前分列值和 upper_bound 相应值相等的索引
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (upper_bound > 0) {
+ for (int i = 0; i < remainder; i++) {
+ result[result_length[0]] = block_size * block_index + i;
+ result_length[0]++;
+ }
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (upper_bound <= 0) {
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * remainder + 7) / 8);
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][index] < value) {
+ result[result_length[0]] = block_size * block_index + index;
+ result_length[0]++;
+ } else if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length++] = index; // 简化赋值逻辑
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (upper_bound <= 0) {
+ encode_pos = (int) (((long) encode_pos * 8 + bw * index + 7) / 8);
+ encode_pos = (int) (((long) encode_pos * 8 + bitWidthList[i] * index + 7) / 8);
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (upper_bound >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] < value) {
+ result[result_length[0]] = block_size * block_index + index_candidate;
+ result_length[0]++;
+ } else if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length++] = index_candidate; // 简化赋值逻辑
+ }
+ }
+ }
+
+ candidate_length = new_length;
+
+ }
+ }
+
+ return encode_pos;
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsTest.java
index 8744b5e..b6ee0bc 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessPartsTest.java
@@ -496,372 +496,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_less_parts_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
-
- int totalSize = data1.size();
- int halfSize = totalSize / 2;
-
- // 创建两个数据列
- int[] col1_data = new int[halfSize];
- int[] col2_data = new int[halfSize];
-
- int max_mul = (int) Math.pow(10, max_decimal);
-
- // 填充第一列
- for (int i = 0; i < halfSize; i++) {
- col1_data[i] = (int) (data1.get(i) * max_mul);
- }
-
- // 填充第二列
- for (int i = 0; i < halfSize; i++) {
- col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
- }
-
- System.out.println(max_decimal);
-
- byte[] encoded_result1 = new byte[col1_data.length * 4];
- byte[] encoded_result2 = new byte[col2_data.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length1 = 0;
- int length2 = 0;
-
- // 编码第一列
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length1 = SubcolumnTest.Encoder(col1_data, block_size, encoded_result1);
- }
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
-
- // 编码第二列
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length2 = SubcolumnTest.Encoder(col2_data, block_size, encoded_result2);
- }
- e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
-
- compressed_size = length1 + length2;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- ratio += ratioTmp;
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- QueryTwoColumns(encoded_result1, encoded_result2,
- queryRange.get(datasetName), queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_less_parts_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal) {
- max_decimal = cur_decimal;
- }
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
-
- int totalSize = data1.size();
- int halfSize = totalSize / 2;
-
- // 创建两个数据列
- int[] col1_data = new int[halfSize];
- int[] col2_data = new int[halfSize];
-
- int max_mul = (int) Math.pow(10, max_decimal);
-
- // 填充第一列
- for (int i = 0; i < halfSize; i++) {
- col1_data[i] = (int) (data1.get(i) * max_mul);
- }
-
- // 填充第二列
- for (int i = 0; i < halfSize; i++) {
- col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
- }
-
- System.out.println(max_decimal);
-
- byte[] encoded_result1 = new byte[col1_data.length * 4];
- byte[] encoded_result2 = new byte[col2_data.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length1 = 0;
- int length2 = 0;
-
- // 编码第一列
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length1 = SubcolumnBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
- }
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
-
- // 编码第二列
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length2 = SubcolumnBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
- }
- e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
-
- compressed_size = length1 + length2;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- ratio += ratioTmp;
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- QueryTwoColumns(encoded_result1, encoded_result2,
- queryRange.get(datasetName), queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessTest.java
index caeef29..330fab8 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryLessTest.java
@@ -202,313 +202,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_less_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_less_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryLessTest.Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMain.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMain.java
new file mode 100644
index 0000000..76e561e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMain.java
@@ -0,0 +1,1307 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryMain {
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/subcolumn_query/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap();
+
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 12000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 1);
+ queryLessRange.put("Gov10", 120000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ int block_size = 512;
+
+ // repeatTime = 1;
+
+ // String outputPath = output_parent_dir + "subcolumn_query_count_block_" +
+ // block_size + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_max.csv";
+
+ // String outputPath = output_parent_dir + "subcolumn_query_greater_new.csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_greater.csv";
+ String outputPath = output_parent_dir + "subcolumn_query_greater_less.csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_less.csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_equal.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnQueryGreaterLessTest.Query(encoded_result, queryRange.get(datasetName), queryLessRange.get(datasetName));
+ // SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryLessTest.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testQueryBeta() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+ // repeatTime = 1;
+
+ int block_size = 512;
+
+ int beta = 3;
+
+ // String outputPath = output_parent_dir + "subcolumn_query_count_block_" +
+ // block_size + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_query_max_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnQueryMaxTest.Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testQueryLong() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ int block_size = 512;
+
+ // String outputPath = output_parent_dir + "subcolumn_long_query_count.csv";
+ String outputPath = output_parent_dir + "subcolumn_long_query_sum2.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnLongQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnLongQuerySum2Test.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ // block size
+ @Test
+ public void test1() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
+ // String output_parent_dir = parent_dir + "result/query_vs_block/";
+
+ // int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ int[] block_size_list = { 512 };
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ // repeatTime = 1;
+
+ for (int block_size : block_size_list) {
+ // String outputPath = output_parent_dir + "subcolumn_query_count_block_" +
+ // block_size + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_query_greater_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryMaxTest.Query(encoded_result);
+ SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("block_size: " + block_size);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ // beta
+ @Test
+ public void test2() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ // int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+ // 17, 18, 19, 20, 21, 22, 23,
+ // 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int[] beta_list = { 3, 4, 5, 6, 7 };
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap();
+
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 12000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 1);
+ queryLessRange.put("Gov10", 120000);
+
+ int repeatTime = 500;
+
+ // repeatTime = 200;
+ // repeatTime = 1;
+
+ for (int beta : beta_list) {
+ // String outputPath = output_parent_dir + "subcolumn_query_count_beta_" + beta
+ // + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_max_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_equal_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_greater_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_greater_new_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_less_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_less_new_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_equal_new_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_count_new_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_max_new_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_sum2_beta_" + beta + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_query_count2_beta_" + beta + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_query_count3_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryCountNewTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryMaxTest.Query(encoded_result);
+ // SubcolumnQueryGreaterTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryGreaterNewTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryLessTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryLessNewTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryEqualNewTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnQueryMaxNewTest.Query(encoded_result);
+ // SubcolumnQueryGreaterLessTest.Query(encoded_result, queryRange.get(datasetName),
+ // queryLessRange.get(datasetName));
+ // SubcolumnQueryCount2Test.Query(encoded_result);
+ SubcolumnQueryCount3Test.Query(encoded_result);
+ // SubcolumnQuerySum2Test.Query(encoded_result, queryRange.get(datasetName));
+
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ // long block size
+ @Test
+ public void test3() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
+ // String output_parent_dir = parent_dir + "result/query_vs_block/";
+
+ // int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
+
+ int[] block_size_list = { 512 };
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 100;
+
+ repeatTime = 500;
+
+ // repeatTime = 1;
+
+ for (int block_size : block_size_list) {
+ // String outputPath = output_parent_dir + "subcolumn_query_count_block_" +
+ // block_size + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_long_query_count_block_" + block_size + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ SubcolumnLongQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("block_size: " + block_size);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ // long beta
+ @Test
+ public void test4() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ // int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+ // 17, 18, 19, 20, 21, 22, 23,
+ // 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int[] beta_list = { 3, 4, 5, 6, 7 };
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 200;
+
+ repeatTime = 500;
+
+ // repeatTime = 1;
+
+ for (int beta : beta_list) {
+ // String outputPath = output_parent_dir + "subcolumn_long_query_count_beta_" + beta
+ // + ".csv";
+ // String outputPath = output_parent_dir + "subcolumn_long_query_max_beta_" + beta + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_long_query_sum2_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnLongQueryCountTest.Query(encoded_result, queryRange.get(datasetName));
+ // SubcolumnLongQueryEqualTest.Query(encoded_result, queryRange.get(datasetName));
+ SubcolumnLongQuerySum2Test.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-column",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+ @Test
+ public void testParts() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ // int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ // 24, 25, 26, 27, 28, 29, 30, 31 };
+
+ int[] beta_list = { 3, 4, 5, 6, 7 };
+
+ int block_size = 512;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ for (int beta : beta_list) {
+ // String outputPath = output_parent_dir + "subcolumn_query_less_parts_beta_" + beta + ".csv";
+ String outputPath = output_parent_dir + "subcolumn_query_less_parts_new_beta_" + beta + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ // 创建两个数据列
+ int[] col1_data = new int[halfSize];
+ int[] col2_data = new int[halfSize];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+
+ // 填充第一列
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ // 填充第二列
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (int) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length1 = 0;
+ int length2 = 0;
+
+ // 编码第一列
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length1 = SubcolumnBetaTest.Encoder(col1_data, block_size, encoded_result1, beta);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ // 编码第二列
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length2 = SubcolumnBetaTest.Encoder(col2_data, block_size, encoded_result2, beta);
+ }
+ e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+
+ compressed_size = length1 + length2;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ ratio += ratioTmp;
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // SubcolumnQueryLessPartsTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ // queryRange.get(datasetName), queryRange.get(datasetName));
+ SubcolumnQueryLessPartsNewTest.QueryTwoColumns(encoded_result1, encoded_result2,
+ queryRange.get(datasetName), queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxNewTest.java
new file mode 100644
index 0000000..f59be04
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxNewTest.java
@@ -0,0 +1,185 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class SubcolumnQueryMaxNewTest {
+
+ public static void Query(byte[] encoded_result) {
+ int encodePos = 0;
+
+ // 解析数据长度
+ int data_length = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ // 解析块大小
+ int block_size = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ // 处理每个块
+ for (int i = 0; i < num_blocks; i++) {
+ encodePos = BlockQueryMax(encoded_result, i, block_size, block_size, encodePos, result, result_length);
+ }
+
+ // 处理剩余部分
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ result[result_length[0]] = value;
+ result_length[0]++;
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockQueryMax(encoded_result, num_blocks, block_size, remainder, encodePos, result, result_length);
+ }
+ }
+
+ public static int BlockQueryMax(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encodePos, int[] result, int[] result_length) {
+ // 读取最小增量
+ int minDelta0 = ((encoded_result[encodePos] & 0xFF) << 24)
+ | ((encoded_result[encodePos + 1] & 0xFF) << 16)
+ | ((encoded_result[encodePos + 2] & 0xFF) << 8)
+ | (encoded_result[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int m = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ // 初始化候选索引
+ int[] candidate_indices = new int[remainder];
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ }
+
+ if (m == 0) {
+ result[result_length[0]] = minDelta0;
+ result_length[0]++;
+ return encodePos;
+ }
+
+ // 获取比特宽度和编码类型
+ int bw = SubcolumnTest.bitWidth(block_size);
+ int beta = encoded_result[encodePos] & 0xFF;
+ encodePos += 1;
+
+ int l = (m + beta - 1) / beta;
+ int[] bitWidthList = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, 1, l, encodingType);
+
+ // 处理每个子列
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+ if (candidate_indices.length == 1) {
+ // Skip if there's only one candidate
+ long bitOffset = encodePos * 8L + (long) bitWidthList[i] * remainder;
+ encodePos = (int) ((bitOffset + 7) / 8);
+ continue;
+ }
+
+ // 处理最大值
+ int maxPart = Integer.MIN_VALUE; // 初始化为最小值
+ int new_length = 0;
+
+ for (int j = 0; j < candidate_indices.length; j++) {
+ int index = candidate_indices[j];
+ int value = SubcolumnTest.bytesToInt(encoded_result, encodePos * 8 + index * bitWidthList[i], bitWidthList[i]);
+
+ if (value > maxPart) {
+ maxPart = value;
+ new_length = 0;
+ candidate_indices[new_length++] = index;
+ } else if (value == maxPart) {
+ candidate_indices[new_length++] = index;
+ }
+ }
+
+ // 更新位移
+ encodePos = (int) (((long) encodePos * 8 + bitWidthList[i] * remainder + 7) / 8);
+ candidate_indices = resizeArray(candidate_indices, new_length); // Resize to keep only valid indices
+ } else {
+ int index = ((encoded_result[encodePos] & 0xFF) << 8) | (encoded_result[encodePos + 1] & 0xFF);
+ encodePos += 2;
+
+ if (candidate_indices.length == 1) {
+ long bitOffset = encodePos * 8L + bw * index;
+ encodePos = (int) ((bitOffset + 7) / 8);
+ bitOffset = encodePos * 8L + (long) bitWidthList[i] * index;
+ encodePos = (int) ((bitOffset + 7) / 8);
+ continue;
+ }
+
+ // 处理RLE部分
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bw, index, run_length);
+ encodePos = SubcolumnTest.decodeBitPacking(encoded_result, encodePos, bitWidthList[i], index, rle_values);
+
+ int maxPart = Integer.MIN_VALUE;
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+
+ for (int j = 0; j < candidate_indices.length; j++) {
+ int index_candidate = candidate_indices[j];
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index && rle_values[rleIndex] > maxPart) {
+ maxPart = rle_values[rleIndex];
+ new_length = 1;
+ candidate_indices[0] = index_candidate; // 初始化为第一个
+ } else if (rleIndex < index && rle_values[rleIndex] == maxPart) {
+ candidate_indices[new_length++] = index_candidate; // 添加当前候选
+ }
+ }
+ candidate_indices = resizeArray(candidate_indices, new_length); // Resize to keep only valid indices
+ }
+ }
+
+ // 记录最大值到结果中
+ result[result_length[0]] = candidate_indices[0];
+ result_length[0]++;
+
+ return encodePos;
+ }
+
+ private static int[] resizeArray(int[] array, int newSize) {
+ int[] newArray = new int[newSize];
+ System.arraycopy(array, 0, newArray, 0, Math.min(array.length, newSize));
+ return newArray;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxTest.java
index 483045b..7af51f2 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQueryMaxTest.java
@@ -227,285 +227,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_max_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_max_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQueryMaxTest.Query(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySortTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySortTest.java
new file mode 100644
index 0000000..71cf96c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySortTest.java
@@ -0,0 +1,492 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class SubcolumnQuerySortTest {
+
+ private static final int[] BLOCK_SIZES = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+
+ private static class BlockSortResult {
+ int nextEncodePos;
+ int[] sortedIndices;
+ }
+
+ private static class Segment {
+ int start;
+ int end;
+
+ Segment(int start, int end) {
+ this.start = start;
+ this.end = end;
+ }
+ }
+
+ // Sort indices inside one block only.
+ public static int[] QuerySortSingleBlock(byte[] encodedResult, int targetBlockId) {
+ int encodePos = 0;
+ int dataLength = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int blockSize = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+ if (targetBlockId < 0 || targetBlockId >= totalBlocks) {
+ return new int[0];
+ }
+
+ for (int i = 0; i < numBlocks; i++) {
+ BlockSortResult result = blockSort(encodedResult, encodePos, i, blockSize, blockSize);
+ encodePos = result.nextEncodePos;
+ if (i == targetBlockId) {
+ return result.sortedIndices;
+ }
+ }
+
+ if (remainder > 0) {
+ int blockId = numBlocks;
+ int base = blockId * blockSize;
+ if (remainder <= 3) {
+ int[] indices = new int[remainder];
+ int[] values = new int[remainder];
+ for (int i = 0; i < remainder; i++) {
+ values[i] = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ indices[i] = base + i;
+ encodePos += 4;
+ }
+ sortSmallByValue(indices, values);
+ if (blockId == targetBlockId) {
+ return indices;
+ }
+ } else {
+ BlockSortResult result = blockSort(encodedResult, encodePos, blockId, blockSize, remainder);
+ encodePos = result.nextEncodePos;
+ if (blockId == targetBlockId) {
+ return result.sortedIndices;
+ }
+ }
+ }
+
+ return new int[0];
+ }
+
+ // Baseline: fully decode all values, then sort one block by value and index.
+ public static int[] QuerySortByDecodeSingleBlock(byte[] encodedResult, int targetBlockId) {
+ int[] decoded = SubcolumnPruneNewTest.Decoder(encodedResult);
+ int dataLength = decoded.length;
+ int blockSize = SubcolumnPruneNewTest.bytes2Integer(encodedResult, 4, 4);
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+ if (targetBlockId < 0 || targetBlockId >= totalBlocks) {
+ return new int[0];
+ }
+
+ int count = targetBlockId < numBlocks ? blockSize : remainder;
+ int start = targetBlockId * blockSize;
+ int[] index = new int[count];
+ for (int i = 0; i < count; i++) {
+ index[i] = start + i;
+ }
+ sortIndicesByValue(index, decoded);
+ return index;
+ }
+
+ private static BlockSortResult blockSort(
+ byte[] encodedResult, int encodePos, int blockId, int blockSize, int rowCount) {
+ BlockSortResult result = new BlockSortResult();
+
+ SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 4);
+ encodePos += 4;
+ int m = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+
+ int[] order = new int[rowCount];
+ for (int i = 0; i < rowCount; i++) {
+ order[i] = i;
+ }
+ int[] values = new int[rowCount];
+
+ if (m == 0) {
+ int[] absIndex = new int[rowCount];
+ int base = blockId * blockSize;
+ for (int i = 0; i < rowCount; i++) {
+ absIndex[i] = base + i;
+ }
+ result.nextEncodePos = encodePos;
+ result.sortedIndices = absIndex;
+ return result;
+ }
+
+ int beta = SubcolumnPruneNewTest.bytes2Integer(encodedResult, encodePos, 1);
+ encodePos += 1;
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 8, l, bitWidthList);
+ int[] encodingType = new int[l];
+ encodePos =
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, encodePos, 2, l, encodingType);
+
+ int bw = SubcolumnPruneNewTest.bitWidth(blockSize);
+ int[] segmentPos = new int[l];
+ int[] runCountList = new int[l];
+ int[] cardinalityList = new int[l];
+ int scanPos = encodePos;
+
+ for (int i = 0; i < l; i++) {
+ segmentPos[i] = scanPos;
+ int type = encodingType[i];
+ int currentBitWidth = bitWidthList[i];
+ if (type == 0) {
+ long bitPos = ((long) scanPos) * 8L + (long) currentBitWidth * rowCount;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else if (type == 1) {
+ int runCount = ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ runCountList[i] = runCount;
+ scanPos += 2;
+ long bitPos = ((long) scanPos) * 8L + (long) runCount * bw;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) runCount * currentBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ } else {
+ int cardinality = ((encodedResult[scanPos] & 0xFF) << 8) | (encodedResult[scanPos + 1] & 0xFF);
+ cardinalityList[i] = cardinality;
+ scanPos += 2;
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ long bitPos = ((long) scanPos) * 8L + (long) cardinality * currentBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ bitPos = ((long) scanPos) * 8L + (long) rowCount * dictBitWidth;
+ scanPos = (int) ((bitPos + 7L) / 8L);
+ }
+ }
+
+ ArrayList<Segment> segments = new ArrayList<>();
+ segments.add(new Segment(0, rowCount));
+
+ for (int i = l - 1; i >= 0; i--) {
+ int[] digitValues =
+ decodeSubcolumnValues(
+ encodedResult,
+ segmentPos[i],
+ encodingType[i],
+ bitWidthList[i],
+ rowCount,
+ bw,
+ runCountList[i],
+ cardinalityList[i]);
+
+ int shift = i * beta;
+ for (int r = 0; r < rowCount; r++) {
+ values[r] |= digitValues[r] << shift;
+ }
+
+ int[] nextOrder = new int[rowCount];
+ ArrayList<Segment> nextSegments = new ArrayList<>();
+
+ int cursor = 0;
+ for (Segment seg : segments) {
+ if (seg.end - seg.start <= 1) {
+ nextOrder[cursor++] = order[seg.start];
+ nextSegments.add(new Segment(cursor - 1, cursor));
+ continue;
+ }
+
+ // beta<=4 in current encoder tests; bucket size 16 is enough.
+ int[] count = new int[16];
+ for (int p = seg.start; p < seg.end; p++) {
+ count[digitValues[order[p]]]++;
+ }
+ int[] start = new int[16];
+ int running = cursor;
+ for (int b = 0; b < 16; b++) {
+ start[b] = running;
+ running += count[b];
+ }
+ int[] write = start.clone();
+ for (int p = seg.start; p < seg.end; p++) {
+ int idx = order[p];
+ int bucket = digitValues[idx];
+ nextOrder[write[bucket]++] = idx;
+ }
+
+ for (int b = 0; b < 16; b++) {
+ if (count[b] > 0) {
+ int s = start[b];
+ int e = s + count[b];
+ nextSegments.add(new Segment(s, e));
+ }
+ }
+ cursor = running;
+ }
+
+ order = nextOrder;
+ segments = nextSegments;
+ }
+
+ int base = blockId * blockSize;
+ int[] sortedIndices = new int[rowCount];
+ for (int i = 0; i < rowCount; i++) {
+ int local = order[i];
+ sortedIndices[i] = base + local;
+ }
+
+ result.nextEncodePos = scanPos;
+ result.sortedIndices = sortedIndices;
+ return result;
+ }
+
+ private static int[] decodeSubcolumnValues(
+ byte[] encodedResult,
+ int segmentPos,
+ int type,
+ int currentBitWidth,
+ int rowCount,
+ int bw,
+ int runCount,
+ int cardinality) {
+ int[] result = new int[rowCount];
+
+ if (type == 0) {
+ long bitStart = ((long) segmentPos) * 8L;
+ for (int i = 0; i < rowCount; i++) {
+ result[i] =
+ SubcolumnPruneNewTest.bytesToInt(
+ encodedResult, (int) (bitStart + (long) i * currentBitWidth), currentBitWidth);
+ }
+ return result;
+ }
+
+ int pos = segmentPos + 2;
+ if (type == 1) {
+ int[] runEnd = new int[runCount];
+ int[] rleValues = new int[runCount];
+ pos = SubcolumnPruneNewTest.decodeBitPacking(encodedResult, pos, bw, runCount, runEnd);
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, pos, currentBitWidth, runCount, rleValues);
+
+ int begin = 0;
+ for (int i = 0; i < runCount; i++) {
+ int end = runEnd[i];
+ int value = rleValues[i];
+ while (begin < end && begin < rowCount) {
+ result[begin++] = value;
+ }
+ }
+ return result;
+ }
+
+ int dictBitWidth = SubcolumnPruneNewTest.bitWidth(cardinality);
+ int[] dictKeyList = new int[cardinality];
+ int[] dictIndexes = new int[rowCount];
+ pos =
+ SubcolumnPruneNewTest.decodeBitPacking(
+ encodedResult, pos, currentBitWidth, cardinality, dictKeyList);
+ SubcolumnPruneNewTest.decodeBitPacking(encodedResult, pos, dictBitWidth, rowCount, dictIndexes);
+
+ for (int i = 0; i < rowCount; i++) {
+ result[i] = dictKeyList[dictIndexes[i]];
+ }
+ return result;
+ }
+
+ private static void sortSmallByValue(int[] indices, int[] values) {
+ for (int i = 0; i < values.length; i++) {
+ int minPos = i;
+ for (int j = i + 1; j < values.length; j++) {
+ if (values[j] < values[minPos]
+ || (values[j] == values[minPos] && indices[j] < indices[minPos])) {
+ minPos = j;
+ }
+ }
+ if (minPos != i) {
+ int tmpV = values[i];
+ values[i] = values[minPos];
+ values[minPos] = tmpV;
+ int tmpI = indices[i];
+ indices[i] = indices[minPos];
+ indices[minPos] = tmpI;
+ }
+ }
+ }
+
+ private static void sortIndicesByValue(int[] indices, int[] values) {
+ for (int i = 0; i < indices.length; i++) {
+ int minPos = i;
+ for (int j = i + 1; j < indices.length; j++) {
+ int vj = values[indices[j]];
+ int vm = values[indices[minPos]];
+ if (vj < vm || (vj == vm && indices[j] < indices[minPos])) {
+ minPos = j;
+ }
+ }
+ if (minPos != i) {
+ int tmp = indices[i];
+ indices[i] = indices[minPos];
+ indices[minPos] = tmp;
+ }
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf('.');
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.length() - decimalIndex - 1;
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ private interface BlockSortRunner {
+ int[] run(byte[] encodedResult, int blockId);
+ }
+
+ private void runSingleBlockSortBenchmark(
+ String outputPath, String algorithmName, BlockSortRunner runner) throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+
+ int repeatTime = 100;
+ System.out.println("Output: " + outputPath);
+ System.out.println("Block sizes: " + java.util.Arrays.toString(BLOCK_SIZES));
+ System.out.println("Repeat time: " + repeatTime);
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Block Size",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int currentDecimal = getDecimalPrecision(fStr);
+ if (currentDecimal > maxDecimal) {
+ maxDecimal = currentDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ if (maxDecimal > 8) {
+ maxDecimal = 8;
+ }
+ System.out.println("maxDecimal: " + maxDecimal);
+
+ int[] dataArr = new int[data.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (int) (data.get(i) * maxMul);
+ }
+
+ for (int blockSize : BLOCK_SIZES) {
+ byte[] encodedResult = new byte[dataArr.length * 8];
+ int length = 0;
+
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnPruneNewTest.Encoder(dataArr, blockSize, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+
+ int blockId = 0;
+ int[] sortedIndex = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ sortedIndex = runner.run(encodedResult, blockId);
+ }
+ end = System.nanoTime();
+ long querySortTime = (end - start) / repeatTime;
+ System.out.println(
+ "blockSize="
+ + blockSize
+ + ", blockPoints: "
+ + (sortedIndex == null ? 0 : sortedIndex.length));
+ double compressionRatio = length / (double) (data.size() * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ algorithmName,
+ String.valueOf(blockSize),
+ String.valueOf(encodeTime),
+ String.valueOf(querySortTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ System.out.println("compressionRatio: " + compressionRatio);
+ }
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testOptimizedSort() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String outputPath = parentDir + "result/" + "subcolumn_adddict_prunenew_query_sort.csv";
+ runSingleBlockSortBenchmark(
+ outputPath,
+ "SubcolumnAddDictPruneNew",
+ SubcolumnQuerySortTest::QuerySortSingleBlock);
+ }
+
+ @Test
+ public void testDecodeSort() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String outputPath = parentDir + "result/" + "subcolumn_adddict_prunenew_query_sort_decode.csv";
+ runSingleBlockSortBenchmark(
+ outputPath,
+ "SubcolumnAddDictPruneNew",
+ SubcolumnQuerySortTest::QuerySortByDecodeSingleBlock);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySum2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySum2Test.java
index f840e90..6b559fd 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySum2Test.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySum2Test.java
@@ -36,7 +36,7 @@
int[] result_length = new int[1];
for (int i = 0; i < num_blocks; i++) {
- encode_pos = BlockQuerySum(encoded_result, i, block_size, block_size, encode_pos, target, result,
+ encode_pos = BlockQuery(encoded_result, i, block_size, block_size, encode_pos, target, result,
result_length);
}
@@ -52,7 +52,7 @@
result_length[0]++;
}
} else {
- encode_pos = BlockQuerySum(encoded_result, num_blocks, block_size, remainder, encode_pos, target,
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size, remainder, encode_pos, target,
result, result_length);
}
@@ -63,7 +63,7 @@
}
- public static int BlockQuerySum(byte[] encoded_result, int block_index, int block_size, int remainder,
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
int encode_pos, int target, int[] result, int[] result_length) {
int[] min_delta = new int[3];
@@ -194,311 +194,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- HashMap<String, Integer> queryRange = new HashMap<>();
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_sum2_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- HashMap<String, Integer> queryRange = new HashMap<>();
-
- queryRange.put("Bird-migration", 2600000);
- queryRange.put("Bitcoin-price", 170000000);
- queryRange.put("City-temp", 700);
- queryRange.put("Dewpoint-temp", 9600);
- queryRange.put("IR-bio-temp", -200);
- queryRange.put("PM10-dust", 2000);
- queryRange.put("Stocks-DE", 90000);
- queryRange.put("Stocks-UK", 30000);
- queryRange.put("Stocks-USA", 6000);
- queryRange.put("Wind-Speed", 60);
- queryRange.put("Wine-Tasting", 10);
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_sum2_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQuerySum2Test.Query(encoded_result, queryRange.get(datasetName));
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySumTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySumTest.java
index e7efa97..c2f7773 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySumTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnQuerySumTest.java
@@ -35,7 +35,7 @@
int[] result_length = new int[1];
for (int i = 0; i < num_blocks; i++) {
- encode_pos = BlockQuerySum(encoded_result, i, block_size, block_size, encode_pos, result,
+ encode_pos = BlockQuery(encoded_result, i, block_size, block_size, encode_pos, result,
result_length);
}
@@ -51,7 +51,7 @@
result_length[0]++;
}
} else {
- encode_pos = BlockQuerySum(encoded_result, num_blocks, block_size, remainder, encode_pos,
+ encode_pos = BlockQuery(encoded_result, num_blocks, block_size, remainder, encode_pos,
result, result_length);
}
@@ -62,7 +62,7 @@
}
- public static int BlockQuerySum(byte[] encoded_result, int block_index, int block_size, int remainder,
+ public static int BlockQuery(byte[] encoded_result, int block_index, int block_size, int remainder,
int encode_pos, int[] result, int[] result_length) {
int[] min_delta = new int[3];
@@ -134,285 +134,4 @@
return encode_pos;
}
- public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
- int decimalIndex = str.indexOf(".");
-
- // 如果没有小数点，精度为0
- if (decimalIndex == -1) {
- return 0;
- }
-
- // 获取小数点后的部分并返回其长度
- return str.substring(decimalIndex + 1).length();
- }
-
- public static String extractFileName(String path) {
- if (path == null || path.isEmpty()) {
- return "";
- }
-
- File file = new File(path);
- String fileName = file.getName();
-
- int dotIndex = fileName.lastIndexOf('.');
-
- if (dotIndex == -1 || dotIndex == 0) {
- return fileName;
- }
-
- return fileName.substring(0, dotIndex);
- }
-
- @Test
- public void testQuery() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_block/";
- // String output_parent_dir = parent_dir + "result/query_vs_block/";
-
- int[] block_size_list = { 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192 };
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int block_size : block_size_list) {
- String outputPath = output_parent_dir + "subcolumn_query_sum_block_" + block_size + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- Query(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("block_size: " + block_size);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
- @Test
- public void testQueryBeta() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String input_parent_dir = parent_dir + "dataset/";
-
- String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
- // String output_parent_dir = parent_dir + "result/query_vs_beta/";
-
- int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
- 24, 25, 26, 27, 28, 29, 30, 31 };
-
- int block_size = 512;
-
- int repeatTime = 200;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
-
- for (int beta : beta_list) {
- String outputPath = output_parent_dir + "subcolumn_query_sum_beta_" + beta + ".csv";
-
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
- File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
-
- for (File file : csvFiles) {
- String datasetName = extractFileName(file.toString());
- System.out.println(datasetName);
-
- InputStream inputStream = Files.newInputStream(file.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Float> data1 = new ArrayList<>();
-
- int max_decimal = 0;
- while (loader.readRecord()) {
- String f_str = loader.getValues()[0];
- if (f_str.isEmpty()) {
- continue;
- }
- int cur_decimal = getDecimalPrecision(f_str);
- if (cur_decimal > max_decimal)
- max_decimal = cur_decimal;
- data1.add(Float.valueOf(f_str));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- int max_mul = (int) Math.pow(10, max_decimal);
- for (int i = 0; i < data1.size(); i++) {
- data2_arr[i] = (int) (data1.get(i) * max_mul);
- }
-
- System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
-
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = SubcolumnBetaTest.Encoder(data2_arr, block_size, encoded_result, beta);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
-
- double ratioTmp;
-
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
-
- System.out.println("Query");
-
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- SubcolumnQuerySumTest.Query(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- String[] record = {
- datasetName,
- "Sub-columns",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
-
- System.out.println("beta: " + beta);
-
- System.out.println(ratio);
- }
-
- writer.close();
- }
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnRLETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnRLETest.java
new file mode 100644
index 0000000..5af5cd8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnRLETest.java
@@ -0,0 +1,840 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnRLETest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int SubcolumnRLE(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ // int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ // if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ // bpBest = true;
+ // break;
+ // }
+ }
+
+ // if (bpBest) {
+ // cost += bpCost;
+ // continue;
+ // }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ // if (bpCost <= rleCost) {
+ // cost += bpCost;
+ // } else {
+ // cost += rleCost;
+ // }
+
+ cost += rleCost;
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ // int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ // if (bw * index + bitWidthList[i] * index >= bpCost) {
+ // break;
+ // }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ // if (bpCost <= rleCost) {
+ // encodingType[i] = 0;
+
+ // encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ // } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ // }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = SubcolumnRLE(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ // String input_parent_dir = parent_dir + "dataset/CMS9";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "subcolumn.csv";
+ String outputPath = output_parent_dir + "subcolumn_rle.csv";
+
+ // int block_size = 512;
+ int block_size = 256;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 9];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ int[] data2_arr_decoded = new int[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ for (int i = 0; i < data2_arr_decoded.length; i++) {
+ // assertEquals(data2_arr[i], data2_arr_decoded[i]);
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSAPHANATest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSAPHANATest.java
new file mode 100644
index 0000000..453112e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSAPHANATest.java
@@ -0,0 +1,1091 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.Stream;
+
+import static java.lang.Math.min;
+import static org.junit.Assert.assertEquals;
+
+public class SubcolumnSAPHANATest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ private static long[] generateTimestampSeries(int totalPoints, double equidistantRatio) {
+ Random random = new Random(42); // 固定种子以确保可重复性
+
+ long currentTime = System.currentTimeMillis();
+ int equidistantPoints = (int) (totalPoints * equidistantRatio);
+ int randomPoints = totalPoints - equidistantPoints;
+
+ // 创建等距部分的时间戳数组
+ long[] equidistantArray = new long[equidistantPoints];
+ for (int i = 0; i < equidistantPoints; i++) {
+ equidistantArray[i] = currentTime;
+ currentTime += 1000; // 每秒一个点
+ }
+
+ // 创建随机部分的时间戳数组
+ long[] randomArray = new long[randomPoints];
+ for (int i = 0; i < randomPoints; i++) {
+ // 随机间隔，从1毫秒到1小时
+ long randomInterval = 1 + random.nextInt(3600000);
+ currentTime += randomInterval;
+ randomArray[i] = currentTime;
+ }
+
+ // 将随机部分的时间戳插入到等距部分中，确保有序
+ long[] result = Arrays.copyOf(equidistantArray, totalPoints);
+
+ for (int i = 0; i < randomArray.length; i++) {
+ // 找到插入位置
+ int insertPos = Arrays.binarySearch(result, 0, equidistantPoints + i, randomArray[i]);
+ if (insertPos < 0) {
+ insertPos = -insertPos - 1; // 转换为插入位置
+ }
+
+ // 移动元素并插入
+ System.arraycopy(result, insertPos, result, insertPos + 1, equidistantPoints + i - insertPos);
+ result[insertPos] = randomArray[i];
+ }
+
+ return result;
+ }
+ @Test
+ public void testSubcolumn() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+ String output_parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_vs_sap_hana/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "sap_hana.csv";
+
+ // int block_size = 512;
+ int block_size = 256;
+
+ // int repeatTime = 100;
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+
+ int totalPoints = 10000; // 10万个点
+ int numTests = 11; // 11个测试用例
+ String[] rate_list = {"0.0","0.1","0.2","0.3","0.4","0.5","0.6","0.7","0.8","0.9","1.0"};
+ // 测试不同分段占比
+ for (int i = 0; i < numTests; i++) {
+ double rate = i * 0.1;
+ String outputPath = output_parent_dir + "subcolumn_" + rate_list[numTests-i-1] + ".csv";
+ System.out.println(outputPath);
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ long[] data2_arr = generateTimestampSeries(totalPoints,rate);
+ byte[] encoded_result = new byte[Long.BYTES*totalPoints];
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnLongTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data2_arr.length * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded =SubcolumnLongTest.Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+
+ String[] record = {
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data2_arr.length),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ writer.close();
+ }
+ }
+
+ @Test
+ public void testSAPHANA() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+// // String parent_dir = "D:/encoding-subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+// // String input_parent_dir = parent_dir + "dataset/CMS9";
+//
+ String output_parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_vs_sap_hana/";
+ // String output_parent_dir = parent_dir + "result/";
+
+ // String outputPath = output_parent_dir + "subcolumn.csv";
+// String outputPath = output_parent_dir + "sap_hana.csv";
+
+ // int block_size = 512;
+ int block_size = 256;
+ String[] rate_list = {"0.0","0.1","0.2","0.3","0.4","0.5","0.6","0.7","0.8","0.9","1.0"};
+ // int repeatTime = 100;
+ int repeatTime = 100;
+
+ // repeatTime = 1;
+
+
+ int totalPoints = 10000; // 10万个点
+ int numTests = 11; // 11个测试用例
+
+ // 测试不同分段占比
+ for (int i = 0; i < numTests; i++) {
+ double rate = i * 0.1;
+ String outputPath = output_parent_dir + "sap_hana_" + rate_list[numTests-1-i] + ".csv";
+ System.out.println(outputPath);
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ long[] data2_arr = generateTimestampSeries(totalPoints,rate);
+ byte[] encoded_result = new byte[Long.BYTES*totalPoints];
+
+
+
+ CompressedData compressed= compress(data2_arr);
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ compressed = compress(data2_arr);
+ length = 8 + 8 + (compressed.tsx.length * 4) + (compressed.ts0Array.length * 8) + (compressed.ts0Indices.length * 4);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data2_arr.length * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ long[] decompressed = decompress(compressed);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+
+ String[] record = {
+ "SAP HANA",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data2_arr.length),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ writer.close();
+ }
+ }
+// @Test
+// public void main() throws IOException {
+//
+// }
+
+ // 压缩数据结构
+ public static class CompressedData {
+ public long ts0; // 基准时间戳
+ public long tsm; // 单位增量（100纳秒为单位）
+ public int[] tsx; // 单位数量数组
+ public long[] ts0Array; // 不规则时间戳数组
+ public int[] ts0Indices; // 不规则时间戳的索引
+
+ public CompressedData(long ts0, long tsm, int[] tsx, long[] ts0Array, int[] ts0Indices) {
+ this.ts0 = ts0;
+ this.tsm = tsm;
+ this.tsx = tsx;
+ this.ts0Array = ts0Array;
+ this.ts0Indices = ts0Indices;
+ }
+ }
+
+ /**
+ * 压缩时间戳数组
+ * @param timeArr 原始时间戳数组（毫秒）
+ * @return 压缩后的数据
+ */
+ public static CompressedData compress(long[] timeArr) {
+ if (timeArr == null || timeArr.length == 0) {
+ return null;
+ }
+
+ // 1. 计算基准时间戳 (ts.b)
+ long ts0 = timeArr[0];
+
+ // 2. 计算最常见的间隔 (转换为100纳秒单位)
+ long mostCommonInterval = findMostCommonInterval(timeArr);
+ long tsm = mostCommonInterval * 10000; // 毫秒转换为100纳秒单位
+
+ // 3. 识别不规则的时间戳
+ List<Long> irregularTimestamps = new ArrayList<>();
+ List<Integer> irregularIndices = new ArrayList<>();
+ int[] tsx = new int[timeArr.length];
+
+ for (int i = 0; i < timeArr.length; i++) {
+ long timestamp = timeArr[i];
+ long expectedTime = ts0 + (i * mostCommonInterval);
+
+ if (timestamp == expectedTime) {
+ // 规则时间戳，计算单位数
+ tsx[i] = i;
+ } else {
+ // 不规则时间戳，记录到ts0Array
+ irregularTimestamps.add(timestamp);
+ irregularIndices.add(i);
+ tsx[i] = -1; // 标记为不规则
+ }
+ }
+
+ // 转换为数组
+ long[] ts0Array = new long[irregularTimestamps.size()];
+ int[] ts0Indices = new int[irregularIndices.size()];
+
+ for (int i = 0; i < irregularTimestamps.size(); i++) {
+ ts0Array[i] = irregularTimestamps.get(i);
+ ts0Indices[i] = irregularIndices.get(i);
+ }
+
+ return new CompressedData(ts0, tsm, tsx, ts0Array, ts0Indices);
+ }
+
+ /**
+ * 解压时间戳数组
+ * @param compressedData 压缩后的数据
+ * @return 原始时间戳数组
+ */
+ public static long[] decompress(CompressedData compressedData) {
+ if (compressedData == null) {
+ return null;
+ }
+
+ int length = compressedData.tsx.length;
+ long[] result = new long[length];
+
+ // 处理不规则时间戳
+ for (int i = 0; i < compressedData.ts0Indices.length; i++) {
+ int index = compressedData.ts0Indices[i];
+ result[index] = compressedData.ts0Array[i];
+ }
+
+ // 处理规则时间戳
+ for (int i = 0; i < length; i++) {
+ if (compressedData.tsx[i] != -1) { // 不是不规则时间戳
+ // 计算时间戳: ts0 + (tsm * tsx) / 10000 (转换为毫秒)
+ result[i] = compressedData.ts0 + (compressedData.tsm * compressedData.tsx[i]) / 10000;
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * 查找最常见的时间间隔
+ * @param timeArr 时间戳数组
+ * @return 最常见的时间间隔（毫秒）
+ */
+ private static long findMostCommonInterval(long[] timeArr) {
+ if (timeArr.length < 2) {
+ return 0;
+ }
+
+ // 计算所有间隔
+ long[] intervals = new long[timeArr.length - 1];
+ for (int i = 1; i < timeArr.length; i++) {
+ intervals[i - 1] = timeArr[i] - timeArr[i - 1];
+ }
+
+ // 找出最常见的间隔
+ long mostCommon = intervals[0];
+ int maxCount = 1;
+
+ for (int i = 0; i < intervals.length; i++) {
+ int count = 0;
+ for (int j = 0; j < intervals.length; j++) {
+ if (intervals[i] == intervals[j]) {
+ count++;
+ }
+ }
+
+ if (count > maxCount) {
+ maxCount = count;
+ mostCommon = intervals[i];
+ }
+ }
+
+ return mostCommon;
+ }
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSum2WithNULLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSum2WithNULLTest.java
new file mode 100644
index 0000000..8c9a3d1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSum2WithNULLTest.java
@@ -0,0 +1,468 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class SubcolumnSum2WithNULLTest {
+
+ public static int Query(byte[] encoded_result, int target) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuerySum(encoded_result, i, block_size, block_size, encode_pos, target, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ } else {
+ encode_pos = BlockQuerySum(encoded_result, num_blocks, block_size, remainder, encode_pos, target,
+ result, result_length);
+ }
+
+ // for (int i = 0; i < result_length[0]; i++) {
+ // System.out.print(result[i] + " ");
+ // }
+ // System.out.println();
+return result_length[0];
+ }
+
+ public static int BlockQuerySum(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int target, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ target -= min_delta[0];
+
+ int[] candidate_indices = new int[remainder];
+ int candidate_length = 0;
+ for (int i = 0; i < remainder; i++) {
+ candidate_indices[i] = i;
+ candidate_length++;
+ }
+
+ if (m == 0) {
+ if (target == 0) {
+ result[result_length[0]] = min_delta[0] * remainder;
+ result_length[0]++;
+ }
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * remainder;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ encode_pos *= 8;
+
+ int new_length = 0;
+ for (int j = 0; j < candidate_length; j++) {
+ int index = candidate_indices[j];
+
+ subcolumnList[i][index] = SubcolumnTest.bytesToInt(encoded_result,
+ encode_pos + index * bitWidthList[i], bitWidthList[i]);
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ if (subcolumnList[i][index] == value) {
+ candidate_indices[new_length] = index;
+ new_length++;
+ }
+ }
+
+ candidate_length = new_length;
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ if (target < 0) {
+ encode_pos *= 8;
+ encode_pos += bw * index;
+ encode_pos = (encode_pos + 7) / 8;
+
+ encode_pos *= 8;
+ encode_pos += bitWidthList[i] * index;
+ encode_pos = (encode_pos + 7) / 8;
+ continue;
+ }
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ int new_length = 0;
+ int rleIndex = 0;
+ int currentPos = 0;
+ int value = (target >> (i * beta)) & ((1 << beta) - 1);
+
+ for (int j = 0; j < candidate_length; j++) {
+ int index_candidate = candidate_indices[j];
+
+ while (rleIndex < index && currentPos + run_length[rleIndex] <= index_candidate) {
+ currentPos += run_length[rleIndex];
+ rleIndex++;
+ }
+
+ if (rleIndex < index) {
+ if (rle_values[rleIndex] == value) {
+ candidate_indices[new_length] = index_candidate;
+ new_length++;
+ }
+ }
+ }
+
+ candidate_length = new_length;
+ }
+ }
+
+ result[result_length[0]] = (min_delta[0] + target) * candidate_length;
+ result_length[0]++;
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ @Test
+ public void testQueryBeta() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/query_sum_null/"; //""D:/encoding-subcolumn/result/";
+
+// int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+// 24, 25, 26, 27, 28, 29, 30, 31 };
+ double[] null_rate_list = {0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1};
+
+ int block_size = 512;
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2600000);
+ queryRange.put("Bitcoin-price", 170000000);
+ queryRange.put("City-temp", 700);
+ queryRange.put("Dewpoint-temp", 9600);
+ queryRange.put("IR-bio-temp", -200);
+ queryRange.put("PM10-dust", 2000);
+ queryRange.put("Stocks-DE", 90000);
+ queryRange.put("Stocks-UK", 30000);
+ queryRange.put("Stocks-USA", 6000);
+ queryRange.put("Wind-Speed", 60);
+ queryRange.put("Wine-Tasting", 10);
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (double null_rate : null_rate_list) {
+ String outputPath = output_parent_dir + "subcolumn_query_sum_null_" + null_rate + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")) continue;
+ if(!queryRange.containsKey(datasetName)) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int nullCountPerBlock = (int) (null_rate * (double) block_size); // 每块中要设置为null的数量
+ int new_arr_length = (int) ((double)(data1.size()/block_size*block_size)*(1-null_rate)+
+ (double)(data1.size()-data1.size()/block_size*block_size)*(1-null_rate));
+ System.out.println("new_arr_length:"+new_arr_length);
+ int[] data2_arr_new = new int[new_arr_length];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ java.util.BitSet bitmap = new java.util.BitSet(data1.size());
+ int new_array_index = 0;
+
+ if(null_rate==1){
+ int[] bitmap_bit = new int[data1.size()];
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ for(int i=0;i<data1.size();i++){
+ bitmap_bit[i] = 0;
+ }
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+ double ratioTmp = 0;
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ String[] decode_values = new String[data1.size()];
+ for(int i=0;i<data1.size();i++){
+ decode_values[i] = "NULL";
+ }
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ continue;
+ } else if(null_rate != 0 && null_rate != 1){
+ for (int blockStart = 0; blockStart < data1.size(); blockStart += block_size) {
+ int blockEnd = Math.min(blockStart + block_size, data1.size());
+ int actualBlockSize = (int) ((double)(blockEnd - blockStart)*(1-null_rate));
+ int actualNullCount = Math.min(nullCountPerBlock, actualBlockSize);
+
+ // 创建当前块的索引列表用于随机选择
+ java.util.List<Integer> indices = new java.util.ArrayList<>();
+ for (int i = blockStart; i < blockEnd; i++) {
+ indices.add(i);
+ }
+
+ // 随机打乱并选择要设置为null的位置
+ java.util.Collections.shuffle(indices);
+ java.util.List<Integer> selectedIndices = new java.util.ArrayList<>();
+ for (int i = 0; i < actualNullCount; i++) {
+ selectedIndices.add(indices.get(i));
+ }
+ java.util.Collections.sort(selectedIndices);
+ int i = 0;
+ int j = blockStart;
+// int nullIndex;
+ while (j < blockEnd) {
+ if (i < actualNullCount && j == selectedIndices.get(i)) {
+ // 这个位置被移除，设置bitmap并跳过
+ bitmap.set(j);
+ i++;
+ } else {
+ // 这个位置保留，复制到新数组
+// if(new_array_index==70000){
+// System.out.println(j/block_size*block_size);
+// System.out.println(j);
+// System.out.println(new_array_index);
+// }
+ data2_arr_new[new_array_index] = data2_arr[j];
+ new_array_index++;
+ if(new_array_index == new_arr_length) break;
+ }
+ j++;
+ }
+ if(new_array_index == new_arr_length) break;
+ }
+ }else {
+ data2_arr_new = data2_arr;
+ }
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ int value_num= 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ value_num= SubcolumnSum2WithNULLTest.Query(encoded_result, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSumWithNULLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSumWithNULLTest.java
new file mode 100644
index 0000000..b42221a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnSumWithNULLTest.java
@@ -0,0 +1,392 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SubcolumnSumWithNULLTest {
+
+ public static void Query(byte[] encoded_result) {
+
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ // 查询结果
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockQuerySum(encoded_result, i, block_size, block_size, encode_pos, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = ((encoded_result[encode_pos] & 0xFF) << 24) |
+ ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+ result[result_length[0]] = value;
+ result_length[0]++;
+ }
+ } else {
+ encode_pos = BlockQuerySum(encoded_result, num_blocks, block_size, remainder, encode_pos,
+ result, result_length);
+ }
+
+ // for (int i = 0; i < result_length[0]; i++) {
+ // System.out.print(result[i] + " ");
+ // }
+ // System.out.println();
+
+ }
+
+ public static int BlockQuerySum(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] result, int[] result_length) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int m = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ if (m == 0) {
+ result[result_length[0]] = min_delta[0];
+ result_length[0]++;
+ return encode_pos;
+ }
+
+ int bw = SubcolumnTest.bitWidth(block_size);
+
+ int beta = encoded_result[encode_pos];
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[] encodingType = new int[l];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ if (type == 0) {
+
+ encode_pos *= 8;
+
+ for (int j = 0; j < remainder; j++) {
+ int value = SubcolumnTest.bytesToInt(encoded_result, encode_pos + j * bitWidthList[i],
+ bitWidthList[i]);
+ result[result_length[0]] += value << (i * beta);
+ }
+
+ encode_pos += remainder * bitWidthList[i];
+ encode_pos = (encode_pos + 7) / 8;
+
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = SubcolumnTest.decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], index,
+ rle_values);
+
+ for (int j = 0; j < index; j++) {
+ int runCount = j == 0 ? run_length[j] : run_length[j] - run_length[j - 1];
+ result[result_length[0]] += (rle_values[j] << (i * beta)) * runCount;
+ // result[result_length[0]] += (rle_values[j] << (i * beta)) * run_length[j];
+ }
+ }
+ }
+
+ result_length[0]++;
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+
+ @Test
+ public void testQueryBeta() throws IOException {
+// String parent_dir = "D:/github/xjz17/subcolumn/";
+//
+// String input_parent_dir = parent_dir + "dataset/";
+//
+// String output_parent_dir = "D:/encoding-subcolumn/result/query_vs_beta/";
+// // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/query_sum_null/"; //""D:/encoding-subcolumn/result/";
+
+// int[] beta_list = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+// 24, 25, 26, 27, 28, 29, 30, 31 };
+ double[] null_rate_list = {0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1};
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+ for (double null_rate : null_rate_list) {
+ String outputPath = output_parent_dir + "subcolumn_query_sum_null_" + null_rate + ".csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ int nullCountPerBlock = (int) (null_rate * (double) block_size); // 每块中要设置为null的数量
+ int new_arr_length = (int) ((double)(data1.size()/block_size*block_size)*(1-null_rate)+
+ (double)(data1.size()-data1.size()/block_size*block_size)*(1-null_rate));
+ System.out.println("new_arr_length:"+new_arr_length);
+ int[] data2_arr_new = new int[new_arr_length];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ java.util.BitSet bitmap = new java.util.BitSet(data1.size());
+ int new_array_index = 0;
+
+ if(null_rate==1){
+ int[] bitmap_bit = new int[data1.size()];
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ for(int i=0;i<data1.size();i++){
+ bitmap_bit[i] = 0;
+ }
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+ double ratioTmp = 0;
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ String[] decode_values = new String[data1.size()];
+ for(int i=0;i<data1.size();i++){
+ decode_values[i] = "NULL";
+ }
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ continue;
+ } else if(null_rate != 0 && null_rate != 1){
+ for (int blockStart = 0; blockStart < data1.size(); blockStart += block_size) {
+ int blockEnd = Math.min(blockStart + block_size, data1.size());
+ int actualBlockSize = (int) ((double)(blockEnd - blockStart)*(1-null_rate));
+ int actualNullCount = Math.min(nullCountPerBlock, actualBlockSize);
+
+ // 创建当前块的索引列表用于随机选择
+ java.util.List<Integer> indices = new java.util.ArrayList<>();
+ for (int i = blockStart; i < blockEnd; i++) {
+ indices.add(i);
+ }
+
+ // 随机打乱并选择要设置为null的位置
+ java.util.Collections.shuffle(indices);
+ java.util.List<Integer> selectedIndices = new java.util.ArrayList<>();
+ for (int i = 0; i < actualNullCount; i++) {
+ selectedIndices.add(indices.get(i));
+ }
+ java.util.Collections.sort(selectedIndices);
+ int i = 0;
+ int j = blockStart;
+// int nullIndex;
+ while (j < blockEnd) {
+ if (i < actualNullCount && j == selectedIndices.get(i)) {
+ // 这个位置被移除，设置bitmap并跳过
+ bitmap.set(j);
+ i++;
+ } else {
+ // 这个位置保留，复制到新数组
+// if(new_array_index==70000){
+// System.out.println(j/block_size*block_size);
+// System.out.println(j);
+// System.out.println(new_array_index);
+// }
+ data2_arr_new[new_array_index] = data2_arr[j];
+ new_array_index++;
+ if(new_array_index == new_arr_length) break;
+ }
+ j++;
+ }
+ if(new_array_index == new_arr_length) break;
+ }
+ }else {
+ data2_arr_new = data2_arr;
+ }
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = SubcolumnTest.Encoder(data2_arr_new, (block_size-nullCountPerBlock), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ SubcolumnSumWithNULLTest.Query(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java
index 668b904..5b9af3a 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnTest.java
@@ -16,8 +16,6 @@
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
-import static org.junit.Assert.assertEquals;
-
public class SubcolumnTest {
public static int bitWidth(int value) {
@@ -247,9 +245,6 @@
int cMin = Integer.MAX_VALUE;
- // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
- // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
- // int[] beta_list = { 1, 2, 3, 4 };
int[] beta_list = { 2, 3, 4 };
int bw = bitWidth(block_size);
@@ -353,15 +348,8 @@
return encode_pos;
}
- // int[] bitWidthList = new int[m];
-
- // int[][] subcolumnList = new int[m][list_length];
-
int l;
- // int betaBest = beta[0];
- // byte betaBest = (byte) beta[0];
-
l = (m + beta[0] - 1) / beta[0];
int[] bitWidthList = new int[l];
@@ -390,12 +378,10 @@
int[] encodingType = new int[l];
- // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
int preTypePos = encode_pos;
encode_pos += (l + 7) / 8;
for (int i = l - 1; i >= 0; i--) {
- // 对于每个分列，计算使用 bit packing 还是 rle
int bpCost = bitWidthList[i] * list_length;
int rleCost = 0;
@@ -679,15 +665,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -709,24 +692,18 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "subcolumn.csv";
int block_size = 512;
- int repeatTime = 100;
-
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
+ int repeatTime = 500;
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -743,7 +720,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -768,6 +744,11 @@
data1.add(Float.valueOf(f_str));
}
inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
int[] data2_arr = new int[data1.size()];
int max_mul = (int) Math.pow(10, max_decimal);
@@ -796,11 +777,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -817,10 +794,6 @@
e = System.nanoTime();
decodeTime += ((e - s) / repeatTime);
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
-
String[] record = {
datasetName,
"Sub-columns",
@@ -837,151 +810,4 @@
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "subcolumn.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- // CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
- // writer.setRecordDelimiter('\n');
-
- // String[] head = {
- // "Input Direction",
- // "Encoding Algorithm",
- // "Encoding Time",
- // "Decoding Time",
- // "Points",
- // "Compressed Size",
- // "Compression Ratio"
- // };
- // writer.writeRecord(head);
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- for (int i = 0; i < data2_arr_decoded.length; i++) {
- assertEquals(data2_arr[i], data2_arr_decoded[i]);
- }
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlpha.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlpha.java
new file mode 100644
index 0000000..7c2b0f4
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlpha.java
@@ -0,0 +1,1399 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+public class SubcolumnVariableAlpha {
+
+ // Used to prevent JIT from optimizing away verification work.
+ private static volatile long VERIFY_SINK = 0L;
+
+ /**
+ * Verify decoded == expected, but do NOT fail the test if it's lossy.
+ * Returns number of mismatched positions (or a large count if length differs).
+ */
+ private static int verifyLosslessNoThrow(int[] expected, int[] actual) {
+ if (expected == actual) {
+ return 0;
+ }
+ if (expected == null || actual == null) {
+ return Integer.MAX_VALUE;
+ }
+ int minLen = Math.min(expected.length, actual.length);
+ int mismatches = Math.abs(expected.length - actual.length);
+ for (int i = 0; i < minLen; i++) {
+ if (expected[i] != actual[i]) {
+ mismatches++;
+ }
+ }
+ return mismatches;
+ }
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ /**
+ * Compute cost and best encoding type for a segment [bitStart, bitEnd) (bitwidth = bitEnd - bitStart).
+ * Returns int[2]: { cost, encodingType } where encodingType is 0=BPE, 1=RLE, 2=DE.
+ */
+ private static int[] costForSegment(
+ int[] x, int x_length, int bitStart, int bitEnd,
+ int[] bpe_cost_single, int[] rle_cost_single, int[] de_cost_single,
+ BitSet[] bitsets, int[] threshold) {
+ int beta = bitEnd - bitStart;
+ if (beta <= 0 || beta > threshold.length) {
+ return new int[] { Integer.MAX_VALUE, 0 };
+ }
+ int currentCost;
+ int bestType = 0;
+
+ int bpCost = 0;
+ int beta_start = bitEnd - 1;
+ while (beta_start >= bitStart && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+ if (beta_start >= bitStart) {
+ bpCost = bpe_cost_single[beta_start] * (beta_start - bitStart + 1);
+ }
+ currentCost = bpCost;
+
+ int rleCostMax = 0;
+ for (int j = bitStart; j < bitEnd && j < rle_cost_single.length; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+ if (rleCostMax < currentCost) {
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = bitStart; j < bitEnd && j < bitsets.length; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ break;
+ }
+ }
+ int rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ bestType = 1;
+ }
+ }
+
+ if (bitEnd <= 32) {
+ int th = threshold[beta - 1];
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> bitStart) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+ if (uniqueValues.size() >= th) {
+ break;
+ }
+ }
+ if (uniqueValues.size() < th) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ bestType = 2;
+ }
+ }
+ }
+
+ return new int[] { currentCost, bestType };
+ }
+
+ /**
+ * Subcolumn with variable bitwidth per subcolumn: each subcolumn can have a different bitwidth.
+ * Uses DP to find the partition of [0, m) into segments (subcolumns) that minimizes a cost model.
+ *
+ * Optimality note: The result is optimal only with respect to our *cost model* (BPE/RLE/DE cost
+ * estimates in bits). The model is approximate: it does not match exact bit-packing (e.g. 8 values
+ * per block), RLE/DE headers, or alignment. We do include the overhead of storing the variable
+ * beta list (1 byte per subcolumn) so that more segments are penalized. Fixed beta can still
+ * win when: (1) the cost model underestimates real size, (2) data suits one beta well, or
+ * (3) block is small so the extra (1+l) bytes for variable betas matter.
+ *
+ * Fills encodingType[0..l-1] and betaOut[0..l-1], returns l (number of subcolumns).
+ */
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size, int[] encodingType, int[] betaOut) {
+
+ if (m == 0) {
+ betaOut[0] = 1;
+ return 1;
+ }
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int[] threshold = getThreshold(block_size);
+
+ BitSet[] bitsets = new BitSet[m];
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ int count = 0;
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+ for (int j = 1; j < x_length; j++) {
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+ bitsets[i].set(j - 1);
+ }
+ }
+ bitsets[i].set(x_length - 1);
+ count++;
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+ }
+
+ int maxBeta = Math.min(m, 32);
+ int[] dp = new int[m + 1];
+ int[] bestBeta = new int[m + 1];
+ int[] bestEncodingType = new int[m + 1];
+ dp[0] = 0;
+ final int BETA_STORAGE_BITS = 8;
+ for (int i = 1; i <= m; i++) {
+ dp[i] = Integer.MAX_VALUE;
+ for (int beta = 1; beta <= Math.min(i, maxBeta); beta++) {
+ int segStart = i - beta;
+ int[] segResult = costForSegment(x, x_length, segStart, i,
+ bpe_cost_single, rle_cost_single, de_cost_single, bitsets, threshold);
+ int segCost = segResult[0];
+ int segType = segResult[1];
+ long total = (long) dp[segStart] + segCost + BETA_STORAGE_BITS;
+ if (total < dp[i]) {
+ dp[i] = (int) total;
+ bestBeta[i] = beta;
+ bestEncodingType[i] = segType;
+ }
+ }
+ }
+
+ int pos = m;
+ int l = 0;
+ int[] revBeta = new int[m];
+ int[] revType = new int[m];
+ while (pos > 0) {
+ int beta = bestBeta[pos];
+ revBeta[l] = beta;
+ revType[l] = bestEncodingType[pos];
+ l++;
+ pos -= beta;
+ }
+ for (int i = 0; i < l; i++) {
+ betaOut[i] = revBeta[l - 1 - i];
+ encodingType[i] = revType[l - 1 - i];
+ }
+ return l;
+ }
+
+ /**
+ * Fixed beta: find the single beta that minimizes total cost over all subcolumns.
+ * Fills encodingType[0..l-1] where l = ceil(m/betaBest). Returns betaBest.
+ */
+ public static int SubcolumnFixed(int[] x, int x_length, int m, int block_size, int[] encodingType) {
+ if (m == 0) {
+ encodingType[0] = 0;
+ return 1;
+ }
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+ int[] threshold = getThreshold(block_size);
+ BitSet[] bitsets = new BitSet[m];
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ int count = 0;
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+ for (int j = 1; j < x_length; j++) {
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+ bitsets[i].set(j - 1);
+ }
+ }
+ bitsets[i].set(x_length - 1);
+ count++;
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+ }
+ int cost1 = 0;
+ for (int i = 0; i < m; i++) {
+ if (bpe_cost_single[i] <= rle_cost_single[i] && bpe_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 0;
+ cost1 += bpe_cost_single[i];
+ } else if (rle_cost_single[i] < bpe_cost_single[i] && rle_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 1;
+ cost1 += rle_cost_single[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += de_cost_single[i];
+ }
+ }
+ int cMin = cost1;
+ int betaBest = 1;
+ for (int beta = 2; beta <= m; beta++) {
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = new int[l];
+ for (int i = 0; i < l; i++) {
+ int currentCost = 0;
+ int bpCost = 0;
+ int beta_start = Math.min(m - 1, (i + 1) * beta - 1);
+ while (beta_start >= i * beta && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+ if (beta_start < i * beta) {
+ beta_start = i * beta;
+ }
+ bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+ currentCost = bpCost;
+ int rleCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+ if (rleCostMax < currentCost) {
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ break;
+ }
+ }
+ int rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+ if (beta <= threshold.length) {
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ break;
+ }
+ }
+ if (uniqueValues.size() < threshold[beta - 1]) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+ cost += currentCost;
+ }
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+ return betaBest;
+ }
+
+ private static int[] getThreshold(int block_size) {
+ switch (block_size) {
+ case 32:
+ return new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ case 64:
+ return new int[] {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ case 128:
+ return new int[] {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ case 256:
+ return new int[] {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ case 512:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ case 1024:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ case 2048:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ case 4096:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ case 8192:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ default:
+ return new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ }
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int l, int block_size, int[] encodingType) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int[] bitWidthList = new int[l];
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(l, encode_pos, encoded_result);
+ encode_pos += 1;
+ for (int i = 0; i < l; i++) {
+ intByte2Bytes(beta[i], encode_pos + i, encoded_result);
+ }
+ encode_pos += l;
+
+ int bw = bitWidth(block_size);
+ int shiftSoFar = 0;
+ for (int i = 0; i < l; i++) {
+ int mask = (1 << beta[i]) - 1;
+ int maxValuePart = 0;
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftSoFar) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ shiftSoFar += beta[i];
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+
+ if (encodingType[i] == 2) {
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ int dict_bit_width = bitWidth(cardinality) ;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ }
+
+ index++;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int l = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+ int[] beta = new int[l];
+ for (int i = 0; i < l; i++) {
+ beta[i] = bytes2Integer(encoded_result, encode_pos + i, 1);
+ }
+ encode_pos += l;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if(type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ encode_pos =decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ int shiftSoFar = 0;
+ for (int i = 0; i < l; i++) {
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftSoFar;
+ }
+ shiftSoFar += beta[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ private static final int TEMP_ENCODE_BUF_SIZE = 256 * 1024;
+
+ private static final class PartitionStats {
+ private final Map<String, Integer> partitionCount = new HashMap<>();
+ private final Map<Integer, Integer> betaHist = new HashMap<>();
+
+ void addPartition(int[] betas, int l) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < l; i++) {
+ if (i > 0) {
+ sb.append(',');
+ }
+ sb.append(betas[i]);
+ betaHist.merge(betas[i], 1, Integer::sum);
+ }
+ partitionCount.merge(sb.toString(), 1, Integer::sum);
+ }
+
+ List<Map.Entry<String, Integer>> topPartitions(int k) {
+ ArrayList<Map.Entry<String, Integer>> entries = new ArrayList<>(partitionCount.entrySet());
+ entries.sort((a, b) -> Integer.compare(b.getValue(), a.getValue()));
+ return entries.subList(0, Math.min(k, entries.size()));
+ }
+
+ List<Map.Entry<Integer, Integer>> betaHistogram() {
+ ArrayList<Map.Entry<Integer, Integer>> entries = new ArrayList<>(betaHist.entrySet());
+ entries.sort(Map.Entry.comparingByKey());
+ return entries;
+ }
+ }
+
+ /*
+ * The following "choose + export per-block optimal partition" helper was used only for writing
+ * partition details into CSV. It's currently disabled in test0(), so we comment it out to keep
+ * this test file clean (no unused warnings). Re-enable if you want the per-block export again.
+ */
+ // private static final class OptimalPartitionResult {
+ // private final int[] betas;
+ // private final int l;
+ // private final int m;
+ // private final boolean useVariable;
+ // private final int sizeVar;
+ // private final int sizeFixed;
+ //
+ // private OptimalPartitionResult(
+ // int[] betas, int l, int m, boolean useVariable, int sizeVar, int sizeFixed) {
+ // this.betas = betas;
+ // this.l = l;
+ // this.m = m;
+ // this.useVariable = useVariable;
+ // this.sizeVar = sizeVar;
+ // this.sizeFixed = sizeFixed;
+ // }
+ // }
+ //
+ // private static OptimalPartitionResult chooseOptimalPartitionForBlock(
+ // int[] dataDelta, int remainder, int block_size) {
+ // int maxValue = 0;
+ // for (int j = 0; j < remainder; j++) {
+ // int v = dataDelta[j];
+ // if (v > maxValue) {
+ // maxValue = v;
+ // }
+ // }
+ // int m = bitWidth(maxValue);
+ // if (m == 0) {
+ // return new OptimalPartitionResult(new int[] {1}, 1, 0, true, 0, 0);
+ // }
+ //
+ // int[] encodingTypeVar = new int[m];
+ // int[] betaOut = new int[m];
+ // int lVar = Subcolumn(dataDelta, remainder, m, block_size, encodingTypeVar, betaOut);
+ // byte[] tempVar = new byte[TEMP_ENCODE_BUF_SIZE];
+ // int sizeVar = SubcolumnEncoder(dataDelta, 0, tempVar, betaOut, lVar, block_size, encodingTypeVar);
+ //
+ // int[] encodingTypeFixed = new int[m];
+ // int betaFixed = SubcolumnFixed(dataDelta, remainder, m, block_size, encodingTypeFixed);
+ // int lFixed = (m + betaFixed - 1) / betaFixed;
+ // int[] betaFixedArr = new int[lFixed];
+ // Arrays.fill(betaFixedArr, betaFixed);
+ // byte[] tempFixed = new byte[TEMP_ENCODE_BUF_SIZE];
+ // int sizeFixed = SubcolumnEncoder(dataDelta, 0, tempFixed, betaFixedArr, lFixed, block_size, encodingTypeFixed);
+ //
+ // if (sizeVar <= sizeFixed) {
+ // return new OptimalPartitionResult(Arrays.copyOf(betaOut, lVar), lVar, m, true, sizeVar, sizeFixed);
+ // } else {
+ // return new OptimalPartitionResult(betaFixedArr, lFixed, m, false, sizeVar, sizeFixed);
+ // }
+ // }
+
+ private static void collectOptimalPartitionForBlock(
+ int[] dataDelta, int remainder, int block_size, PartitionStats stats) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ int v = dataDelta[j];
+ if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ int m = bitWidth(maxValue);
+ if (m == 0) {
+ stats.addPartition(new int[] {1}, 1);
+ return;
+ }
+
+ int[] encodingTypeVar = new int[m];
+ int[] betaOut = new int[m];
+ int lVar = Subcolumn(dataDelta, remainder, m, block_size, encodingTypeVar, betaOut);
+ byte[] tempVar = new byte[TEMP_ENCODE_BUF_SIZE];
+ int sizeVar = SubcolumnEncoder(dataDelta, 0, tempVar, betaOut, lVar, block_size, encodingTypeVar);
+
+ int[] encodingTypeFixed = new int[m];
+ int betaFixed = SubcolumnFixed(dataDelta, remainder, m, block_size, encodingTypeFixed);
+ int lFixed = (m + betaFixed - 1) / betaFixed;
+ int[] betaFixedArr = new int[lFixed];
+ Arrays.fill(betaFixedArr, betaFixed);
+ byte[] tempFixed = new byte[TEMP_ENCODE_BUF_SIZE];
+ int sizeFixed = SubcolumnEncoder(dataDelta, 0, tempFixed, betaFixedArr, lFixed, block_size, encodingTypeFixed);
+
+ if (sizeVar <= sizeFixed) {
+ stats.addPartition(betaOut, lVar);
+ } else {
+ stats.addPartition(betaFixedArr, lFixed);
+ }
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ if (m == 0) {
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos, encoded_result,
+ beta, 1, block_size, new int[] {0});
+ return encode_pos;
+ }
+
+ int[] encodingTypeVar = new int[Math.max(m, 1)];
+ int[] betaOut = new int[Math.max(m, 1)];
+ int lVar = Subcolumn(data_delta, remainder, m, block_size, encodingTypeVar, betaOut);
+
+ byte[] tempVar = new byte[TEMP_ENCODE_BUF_SIZE];
+ int posVar = SubcolumnEncoder(data_delta, 0, tempVar, betaOut, lVar, block_size, encodingTypeVar);
+ int sizeVar = posVar;
+
+ int[] encodingTypeFixed = new int[Math.max(m, 1)];
+ int betaFixed = SubcolumnFixed(data_delta, remainder, m, block_size, encodingTypeFixed);
+ int lFixed = (m + betaFixed - 1) / betaFixed;
+ int[] betaFixedArr = new int[33];
+ for (int i = 0; i < lFixed; i++) {
+ betaFixedArr[i] = betaFixed;
+ }
+
+ byte[] tempFixed = new byte[TEMP_ENCODE_BUF_SIZE];
+ int posFixed = SubcolumnEncoder(data_delta, 0, tempFixed, betaFixedArr, lFixed, block_size, encodingTypeFixed);
+ int sizeFixed = posFixed;
+
+ if (sizeVar <= sizeFixed) {
+ System.arraycopy(tempVar, 0, encoded_result, encode_pos, sizeVar);
+ encode_pos += sizeVar;
+ } else {
+ System.arraycopy(tempFixed, 0, encoded_result, encode_pos, sizeFixed);
+ encode_pos += sizeFixed;
+ }
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[33];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_variable_alpha.csv";
+ // NOTE: Subcolumn-partition CSV export is currently disabled (commented out).
+ // String partitionOutputPath = output_parent_dir + "subcolumn_variable_alpha_optimal_partition.csv";
+ // String blockPartitionOutputPath = output_parent_dir + "subcolumn_variable_alpha_optimal_partition_per_block.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ // CsvWriter partitionWriter = new CsvWriter(partitionOutputPath, ',', StandardCharsets.UTF_8);
+ // partitionWriter.setRecordDelimiter('\n');
+ //
+ // CsvWriter blockPartitionWriter = new CsvWriter(blockPartitionOutputPath, ',', StandardCharsets.UTF_8);
+ // blockPartitionWriter.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ // String[] partitionHead = {
+ // "Dataset",
+ // "Block Size",
+ // "Top Partition Betas",
+ // "Blocks (Top Partition)",
+ // "Subcolumn Index",
+ // "Beta (bits)"
+ // };
+ // partitionWriter.writeRecord(partitionHead);
+ //
+ // String[] blockPartitionHead = {
+ // "Dataset",
+ // "Block Size",
+ // "Block Index",
+ // "Points In Block",
+ // "m (bitWidth(maxDelta))",
+ // "Chosen Scheme",
+ // "Encoded Size Var (bytes)",
+ // "Encoded Size Fixed (bytes)",
+ // "Betas",
+ // "Subcolumn Index",
+ // "Beta (bits)"
+ // };
+ // blockPartitionWriter.writeRecord(blockPartitionHead);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ // partitionWriter.close();
+ // blockPartitionWriter.close();
+ throw new IOException("No csv files found under: " + input_parent_dir);
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ // reset verification sink per dataset (kept to avoid JIT removing verification work)
+ VERIFY_SINK = 0L;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 8) {
+ max_decimal = 8;
+ }
+
+ int[] data2_arr = new int[data1.size()];
+
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ // Collect "optimal" subcolumn partitions (chosen between variable vs fixed).
+ PartitionStats stats = new PartitionStats();
+ int dataLength = data2_arr.length;
+ int numBlocks = dataLength / block_size;
+ int remainderPoints = dataLength % block_size;
+ for (int bi = 0; bi < numBlocks; bi++) {
+ int[] min_delta = new int[3];
+ int[] data_delta = getAbsDeltaTsBlock(data2_arr, bi, block_size, block_size, min_delta);
+ collectOptimalPartitionForBlock(data_delta, block_size, block_size, stats);
+ // Per-block partition export (disabled)
+ // OptimalPartitionResult r = chooseOptimalPartitionForBlock(data_delta, block_size, block_size);
+ // StringBuilder betasSb = new StringBuilder();
+ // for (int i = 0; i < r.l; i++) {
+ // if (i > 0) {
+ // betasSb.append(',');
+ // }
+ // betasSb.append(r.betas[i]);
+ // }
+ // String betasStr = betasSb.toString();
+ // String scheme = r.useVariable ? "VARIABLE" : "FIXED";
+ // for (int si = 0; si < r.l; si++) {
+ // String[] row = {
+ // datasetName,
+ // String.valueOf(block_size),
+ // String.valueOf(bi),
+ // String.valueOf(block_size),
+ // String.valueOf(r.m),
+ // scheme,
+ // String.valueOf(r.sizeVar),
+ // String.valueOf(r.sizeFixed),
+ // betasStr,
+ // String.valueOf(si),
+ // String.valueOf(r.betas[si])
+ // };
+ // blockPartitionWriter.writeRecord(row);
+ // }
+ }
+ if (remainderPoints > 3) {
+ int[] min_delta = new int[3];
+ int[] data_delta = getAbsDeltaTsBlock(data2_arr, numBlocks, block_size, remainderPoints, min_delta);
+ collectOptimalPartitionForBlock(data_delta, remainderPoints, block_size, stats);
+ // Per-block partition export for remainder block (disabled)
+ // OptimalPartitionResult r = chooseOptimalPartitionForBlock(data_delta, remainderPoints, block_size);
+ // StringBuilder betasSb = new StringBuilder();
+ // for (int i = 0; i < r.l; i++) {
+ // if (i > 0) {
+ // betasSb.append(',');
+ // }
+ // betasSb.append(r.betas[i]);
+ // }
+ // String betasStr = betasSb.toString();
+ // String scheme = r.useVariable ? "VARIABLE" : "FIXED";
+ // for (int si = 0; si < r.l; si++) {
+ // String[] row = {
+ // datasetName,
+ // String.valueOf(block_size),
+ // String.valueOf(numBlocks),
+ // String.valueOf(remainderPoints),
+ // String.valueOf(r.m),
+ // scheme,
+ // String.valueOf(r.sizeVar),
+ // String.valueOf(r.sizeFixed),
+ // betasStr,
+ // String.valueOf(si),
+ // String.valueOf(r.betas[si])
+ // };
+ // blockPartitionWriter.writeRecord(row);
+ // }
+ }
+
+ System.out.println("Optimal subcolumn partitions (top 5):");
+ for (Map.Entry<String, Integer> e1 : stats.topPartitions(5)) {
+ System.out.println(" betas=[" + e1.getKey() + "], blocks=" + e1.getValue());
+ }
+ System.out.println("Beta histogram (beta -> count):");
+ for (Map.Entry<Integer, Integer> e2 : stats.betaHistogram()) {
+ System.out.println(" " + e2.getKey() + " -> " + e2.getValue());
+ }
+
+ // Export the top-1 (most frequent) "optimal" partition's betas (disabled).
+ // List<Map.Entry<String, Integer>> top1 = stats.topPartitions(1);
+ // if (!top1.isEmpty()) {
+ // String betasStr = top1.get(0).getKey();
+ // int blocks = top1.get(0).getValue();
+ // if (betasStr != null && !betasStr.isEmpty()) {
+ // String[] parts = betasStr.split(",");
+ // for (int si = 0; si < parts.length; si++) {
+ // String betaStr = parts[si].trim();
+ // if (betaStr.isEmpty()) {
+ // continue;
+ // }
+ // String[] row = {
+ // datasetName,
+ // String.valueOf(block_size),
+ // betasStr,
+ // String.valueOf(blocks),
+ // String.valueOf(si),
+ // betaStr
+ // };
+ // partitionWriter.writeRecord(row);
+ // }
+ // } else {
+ // String[] row = {
+ // datasetName,
+ // String.valueOf(block_size),
+ // betasStr == null ? "" : betasStr,
+ // String.valueOf(blocks),
+ // "0",
+ // "1"
+ // };
+ // partitionWriter.writeRecord(row);
+ // }
+ // }
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int[] decoded = Decoder(encoded_result);
+ // Include lossless verification in decode time (intentionally adds overhead).
+ VERIFY_SINK += verifyLosslessNoThrow(data2_arr, decoded);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (Dictionary)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ // Make VERIFY_SINK observable so static-analysis doesn't mark it unused.
+ System.out.println("verify_mismatches=" + VERIFY_SINK);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ // partitionWriter.close();
+ // blockPartitionWriter.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlphaSignExpMass.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlphaSignExpMass.java
new file mode 100644
index 0000000..f354a7e
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnVariableAlphaSignExpMass.java
@@ -0,0 +1,1336 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.BitSet;
+import java.util.Set;
+
+public class SubcolumnVariableAlphaSignExpMass {
+
+ public static int bitWidth(int value) {
+ if (value == 0) {
+ return 1;
+ }
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidthLong(long value) {
+ if (value == 0L) {
+ return 1;
+ }
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ private static int readBit(byte[] in, int bitPos) {
+ return bytesToBool(in, bitPos) ? 1 : 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long v, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (v >>> 56);
+ cur_byte[encode_pos + 1] = (byte) (v >>> 48);
+ cur_byte[encode_pos + 2] = (byte) (v >>> 40);
+ cur_byte[encode_pos + 3] = (byte) (v >>> 32);
+ cur_byte[encode_pos + 4] = (byte) (v >>> 24);
+ cur_byte[encode_pos + 5] = (byte) (v >>> 16);
+ cur_byte[encode_pos + 6] = (byte) (v >>> 8);
+ cur_byte[encode_pos + 7] = (byte) (v);
+ }
+
+ public static long bytes2Long(byte[] encoded, int start) {
+ long v = 0;
+ for (int i = 0; i < 8; i++) {
+ v = (v << 8) | (encoded[start + i] & 0xFFL);
+ }
+ return v;
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static int bitPackLong(long[] numbers, int bitWidth, int encodePos, byte[] out, int numValues) {
+ int bitPos = encodePos * 8;
+ int bytes = (bitWidth * numValues + 7) / 8;
+ Arrays.fill(out, encodePos, encodePos + bytes, (byte) 0);
+ for (int i = 0; i < numValues; i++) {
+ long v = numbers[i];
+ for (int k = bitWidth - 1; k >= 0; k--) {
+ boolToBytes(((v >>> k) & 1L) != 0, out, bitPos++);
+ }
+ }
+ return encodePos + bytes;
+ }
+
+ private static int bitUnpackLong(byte[] in, int encodePos, int bitWidth, int numValues, long[] out) {
+ int bitPos = encodePos * 8;
+ for (int i = 0; i < numValues; i++) {
+ long v = 0L;
+ for (int k = 0; k < bitWidth; k++) {
+ v = (v << 1) | readBit(in, bitPos++);
+ }
+ out[i] = v;
+ }
+ return encodePos + (bitWidth * numValues + 7) / 8;
+ }
+
+ // Column codec: 0=BP, 1=RLE, 2=Dict
+ private static int encodeLongColumn(long[] values, int n, int encodePos, byte[] out, int bitWidth) {
+ // costs in bits
+ int bpCost = bitWidth * n + 16; // header approx
+ int runs = 1;
+ for (int i = 1; i < n; i++) {
+ if (values[i] != values[i - 1]) {
+ runs++;
+ }
+ }
+ int rleCost = runs * (bitWidth + bitWidth(n)) + 32; // header approx
+
+ Map<Long, Integer> dictMap = new HashMap<>();
+ for (int i = 0; i < n; i++) {
+ dictMap.put(values[i], 0);
+ }
+ int cardinality = dictMap.size();
+ int dictBw = bitWidth(cardinality);
+ int dictCost = dictBw * n + cardinality * bitWidth + 64;
+
+ int type = 0;
+ int best = bpCost;
+ if (rleCost < best) {
+ best = rleCost;
+ type = 1;
+ }
+ if (dictCost < best) {
+ best = dictCost;
+ type = 2;
+ }
+
+ out[encodePos++] = (byte) type;
+ out[encodePos++] = (byte) bitWidth;
+
+ if (type == 0) {
+ return bitPackLong(values, bitWidth, encodePos, out, n);
+ }
+ if (type == 1) {
+ int2Bytes(runs, encodePos, out);
+ encodePos += 4;
+ int posBw = bitWidth(n);
+ out[encodePos++] = (byte) posBw;
+
+ int idx = 0;
+ int[] runEnds = new int[runs];
+ long[] runVals = new long[runs];
+ long prev = values[0];
+ for (int i = 1; i < n; i++) {
+ if (values[i] != prev) {
+ runEnds[idx] = i;
+ runVals[idx] = prev;
+ idx++;
+ prev = values[i];
+ }
+ }
+ runEnds[idx] = n;
+ runVals[idx] = prev;
+
+ encodePos = bitPacking(runEnds, posBw, encodePos, out, runs);
+ return bitPackLong(runVals, bitWidth, encodePos, out, runs);
+ }
+ if (type == 2) {
+ int2Bytes(cardinality, encodePos, out);
+ encodePos += 4;
+ out[encodePos++] = (byte) dictBw;
+ // sorted dict
+ long[] dict = new long[cardinality];
+ int di = 0;
+ for (Long v : dictMap.keySet()) {
+ dict[di++] = v;
+ }
+ Arrays.sort(dict);
+ Map<Long, Integer> valueToCode = new HashMap<>();
+ for (int i = 0; i < cardinality; i++) {
+ valueToCode.put(dict[i], i);
+ }
+ long[] codes = new long[n];
+ for (int i = 0; i < n; i++) {
+ codes[i] = valueToCode.get(values[i]);
+ }
+ encodePos = bitPackLong(dict, bitWidth, encodePos, out, cardinality);
+ return bitPackLong(codes, dictBw, encodePos, out, n);
+ }
+ throw new IllegalStateException("Unknown column encoding type: " + type);
+ }
+
+ private static int decodeLongColumn(byte[] in, int n, int encodePos, long[] out, int[] bitWidthOut) {
+ int type = in[encodePos++] & 0xFF;
+ int bitWidth = in[encodePos++] & 0xFF;
+ bitWidthOut[0] = bitWidth;
+ if (type == 0) {
+ return bitUnpackLong(in, encodePos, bitWidth, n, out);
+ }
+ if (type == 1) {
+ int runs = bytes2Integer(in, encodePos, 4);
+ encodePos += 4;
+ int posBw = in[encodePos++] & 0xFF;
+ int[] runEnds = new int[runs];
+ encodePos = decodeBitPacking(in, encodePos, posBw, runs, runEnds);
+ long[] runVals = new long[runs];
+ encodePos = bitUnpackLong(in, encodePos, bitWidth, runs, runVals);
+ int cur = 0;
+ for (int r = 0; r < runs; r++) {
+ int end = runEnds[r];
+ long v = runVals[r];
+ while (cur < end) {
+ out[cur++] = v;
+ }
+ }
+ return encodePos;
+ }
+ if (type == 2) {
+ int cardinality = bytes2Integer(in, encodePos, 4);
+ encodePos += 4;
+ int dictBw = in[encodePos++] & 0xFF;
+ long[] dict = new long[cardinality];
+ encodePos = bitUnpackLong(in, encodePos, bitWidth, cardinality, dict);
+ long[] codes = new long[n];
+ encodePos = bitUnpackLong(in, encodePos, dictBw, n, codes);
+ for (int i = 0; i < n; i++) {
+ out[i] = dict[(int) codes[i]];
+ }
+ return encodePos;
+ }
+ throw new IllegalStateException("Unknown column encoding type: " + type);
+ }
+
+ private static int encodeIntColumn(int[] values, int n, int encodePos, byte[] out, int bitWidth) {
+ long[] tmp = new long[n];
+ for (int i = 0; i < n; i++) {
+ tmp[i] = values[i] & 0xFFFFFFFFL;
+ }
+ return encodeLongColumn(tmp, n, encodePos, out, bitWidth);
+ }
+
+ private static int decodeIntColumn(byte[] in, int n, int encodePos, int[] out, int[] bitWidthOut) {
+ long[] tmp = new long[n];
+ int pos = decodeLongColumn(in, n, encodePos, tmp, bitWidthOut);
+ for (int i = 0; i < n; i++) {
+ out[i] = (int) tmp[i];
+ }
+ return pos;
+ }
+
+ private static int encodeIeee754Columns(double[] values, int n, int encodePos, byte[] out) {
+ int2Bytes(n, encodePos, out);
+ encodePos += 4;
+
+ int[] sign = new int[n];
+ int[] exp = new int[n];
+ long[] mantissa = new long[n];
+ for (int i = 0; i < n; i++) {
+ long bits = Double.doubleToRawLongBits(values[i]);
+ sign[i] = (int) ((bits >>> 63) & 1L);
+ exp[i] = (int) ((bits >>> 52) & 0x7FFL);
+ mantissa[i] = bits & ((1L << 52) - 1);
+ }
+
+ encodePos = encodeIntColumn(sign, n, encodePos, out, 1);
+ encodePos = encodeIntColumn(exp, n, encodePos, out, 11);
+ encodePos = encodeLongColumn(mantissa, n, encodePos, out, 52);
+ return encodePos;
+ }
+
+ private static double[] decodeIeee754Columns(byte[] in, int encodePos) {
+ int n = bytes2Integer(in, encodePos, 4);
+ encodePos += 4;
+
+ int[] sign = new int[n];
+ int[] exp = new int[n];
+ long[] mantissa = new long[n];
+ int[] bw = new int[1];
+
+ encodePos = decodeIntColumn(in, n, encodePos, sign, bw);
+ encodePos = decodeIntColumn(in, n, encodePos, exp, bw);
+ encodePos = decodeLongColumn(in, n, encodePos, mantissa, bw);
+
+ double[] out = new double[n];
+ for (int i = 0; i < n; i++) {
+ long bits = (((long) sign[i]) << 63) | (((long) exp[i] & 0x7FFL) << 52) | (mantissa[i] & ((1L << 52) - 1));
+ out[i] = Double.longBitsToDouble(bits);
+ }
+ return out;
+ }
+
+ /**
+ * Compute cost and best encoding type for a segment [bitStart, bitEnd) (bitwidth = bitEnd - bitStart).
+ * Returns int[2]: { cost, encodingType } where encodingType is 0=BPE, 1=RLE, 2=DE.
+ */
+ private static int[] costForSegment(
+ int[] x, int x_length, int bitStart, int bitEnd,
+ int[] bpe_cost_single, int[] rle_cost_single, int[] de_cost_single,
+ BitSet[] bitsets, int[] threshold) {
+ int beta = bitEnd - bitStart;
+ if (beta <= 0 || beta > threshold.length) {
+ return new int[] { Integer.MAX_VALUE, 0 };
+ }
+ int currentCost;
+ int bestType = 0;
+
+ int bpCost = 0;
+ int beta_start = bitEnd - 1;
+ while (beta_start >= bitStart && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+ if (beta_start >= bitStart) {
+ bpCost = bpe_cost_single[beta_start] * (beta_start - bitStart + 1);
+ }
+ currentCost = bpCost;
+
+ int rleCostMax = 0;
+ for (int j = bitStart; j < bitEnd && j < rle_cost_single.length; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+ if (rleCostMax < currentCost) {
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = bitStart; j < bitEnd && j < bitsets.length; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ break;
+ }
+ }
+ int rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ bestType = 1;
+ }
+ }
+
+ if (bitEnd <= 32) {
+ int th = threshold[beta - 1];
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> bitStart) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+ if (uniqueValues.size() >= th) {
+ break;
+ }
+ }
+ if (uniqueValues.size() < th) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ bestType = 2;
+ }
+ }
+ }
+
+ return new int[] { currentCost, bestType };
+ }
+
+ /**
+ * Subcolumn with variable bitwidth per subcolumn: each subcolumn can have a different bitwidth.
+ * Uses DP to find the partition of [0, m) into segments (subcolumns) that minimizes a cost model.
+ *
+ * Optimality note: The result is optimal only with respect to our *cost model* (BPE/RLE/DE cost
+ * estimates in bits). The model is approximate: it does not match exact bit-packing (e.g. 8 values
+ * per block), RLE/DE headers, or alignment. We do include the overhead of storing the variable
+ * beta list (1 byte per subcolumn) so that more segments are penalized. Fixed beta can still
+ * win when: (1) the cost model underestimates real size, (2) data suits one beta well, or
+ * (3) block is small so the extra (1+l) bytes for variable betas matter.
+ *
+ * Fills encodingType[0..l-1] and betaOut[0..l-1], returns l (number of subcolumns).
+ */
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size, int[] encodingType, int[] betaOut) {
+
+ if (m == 0) {
+ betaOut[0] = 1;
+ return 1;
+ }
+
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+
+ int[] threshold = getThreshold(block_size);
+
+ BitSet[] bitsets = new BitSet[m];
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ int count = 0;
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+ for (int j = 1; j < x_length; j++) {
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+ bitsets[i].set(j - 1);
+ }
+ }
+ bitsets[i].set(x_length - 1);
+ count++;
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+ }
+
+ int maxBeta = Math.min(m, 32);
+ int[] dp = new int[m + 1];
+ int[] bestBeta = new int[m + 1];
+ int[] bestEncodingType = new int[m + 1];
+ dp[0] = 0;
+ final int BETA_STORAGE_BITS = 8;
+ for (int i = 1; i <= m; i++) {
+ dp[i] = Integer.MAX_VALUE;
+ for (int beta = 1; beta <= Math.min(i, maxBeta); beta++) {
+ int segStart = i - beta;
+ int[] segResult = costForSegment(x, x_length, segStart, i,
+ bpe_cost_single, rle_cost_single, de_cost_single, bitsets, threshold);
+ int segCost = segResult[0];
+ int segType = segResult[1];
+ long total = (long) dp[segStart] + segCost + BETA_STORAGE_BITS;
+ if (total < dp[i]) {
+ dp[i] = (int) total;
+ bestBeta[i] = beta;
+ bestEncodingType[i] = segType;
+ }
+ }
+ }
+
+ int pos = m;
+ int l = 0;
+ int[] revBeta = new int[m];
+ int[] revType = new int[m];
+ while (pos > 0) {
+ int beta = bestBeta[pos];
+ revBeta[l] = beta;
+ revType[l] = bestEncodingType[pos];
+ l++;
+ pos -= beta;
+ }
+ for (int i = 0; i < l; i++) {
+ betaOut[i] = revBeta[l - 1 - i];
+ encodingType[i] = revType[l - 1 - i];
+ }
+ return l;
+ }
+
+ /**
+ * Fixed beta: find the single beta that minimizes total cost over all subcolumns.
+ * Fills encodingType[0..l-1] where l = ceil(m/betaBest). Returns betaBest.
+ */
+ public static int SubcolumnFixed(int[] x, int x_length, int m, int block_size, int[] encodingType) {
+ if (m == 0) {
+ encodingType[0] = 0;
+ return 1;
+ }
+ int[] bpe_cost_single = new int[m];
+ int[] rle_cost_single = new int[m];
+ int[] de_cost_single = new int[m];
+ int[] threshold = getThreshold(block_size);
+ BitSet[] bitsets = new BitSet[m];
+ for (int i = 0; i < m; i++) {
+ bitsets[i] = new BitSet(x_length);
+ }
+ for (int i = 0; i < m; i++) {
+ int current_value = (x[0] >> i) & 1;
+ if (current_value == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ int count = 0;
+ de_cost_single[i] = x_length * 1 + 2 * 1;
+ for (int j = 1; j < x_length; j++) {
+ int subcolumn_ij = (x[j] >> i) & 1;
+ if (subcolumn_ij == 1) {
+ bpe_cost_single[i] = x_length;
+ }
+ if (subcolumn_ij != current_value) {
+ count++;
+ current_value = subcolumn_ij;
+ de_cost_single[i] = x_length * 2 + 2 * 1;
+ bitsets[i].set(j - 1);
+ }
+ }
+ bitsets[i].set(x_length - 1);
+ count++;
+ rle_cost_single[i] = count * (1 + bitWidth(x_length));
+ }
+ int cost1 = 0;
+ for (int i = 0; i < m; i++) {
+ if (bpe_cost_single[i] <= rle_cost_single[i] && bpe_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 0;
+ cost1 += bpe_cost_single[i];
+ } else if (rle_cost_single[i] < bpe_cost_single[i] && rle_cost_single[i] <= de_cost_single[i]) {
+ encodingType[i] = 1;
+ cost1 += rle_cost_single[i];
+ } else {
+ encodingType[i] = 2;
+ cost1 += de_cost_single[i];
+ }
+ }
+ int cMin = cost1;
+ int betaBest = 1;
+ for (int beta = 2; beta <= m; beta++) {
+ int l = (m + beta - 1) / beta;
+ int cost = 0;
+ int[] encodingTypeTemp = new int[l];
+ for (int i = 0; i < l; i++) {
+ int currentCost = 0;
+ int bpCost = 0;
+ int beta_start = Math.min(m - 1, (i + 1) * beta - 1);
+ while (beta_start >= i * beta && bpe_cost_single[beta_start] == 0) {
+ beta_start--;
+ }
+ if (beta_start < i * beta) {
+ beta_start = i * beta;
+ }
+ bpCost = bpe_cost_single[beta_start] * (beta_start - i * beta + 1);
+ currentCost = bpCost;
+ int rleCostMax = 0;
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ if (rle_cost_single[j] > rleCostMax) {
+ rleCostMax = rle_cost_single[j];
+ }
+ }
+ if (rleCostMax < currentCost) {
+ BitSet mergedBitSet = new BitSet(x_length);
+ for (int j = i * beta; j < (i + 1) * beta && j < m; j++) {
+ mergedBitSet.or(bitsets[j]);
+ if (mergedBitSet.cardinality() >= currentCost) {
+ break;
+ }
+ }
+ int rleCost = mergedBitSet.cardinality() * (beta + bitWidth(x_length));
+ if (rleCost < currentCost) {
+ currentCost = rleCost;
+ encodingTypeTemp[i] = 1;
+ }
+ }
+ if (beta <= threshold.length) {
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < x_length; j++) {
+ int currentNumber = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ uniqueValues.add(currentNumber);
+ if (uniqueValues.size() >= threshold[beta - 1]) {
+ break;
+ }
+ }
+ if (uniqueValues.size() < threshold[beta - 1]) {
+ int deCost = x_length * bitWidth(uniqueValues.size()) + uniqueValues.size() * beta;
+ if (deCost < currentCost) {
+ currentCost = deCost;
+ encodingTypeTemp[i] = 2;
+ }
+ }
+ }
+ cost += currentCost;
+ }
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ System.arraycopy(encodingTypeTemp, 0, encodingType, 0, l);
+ }
+ }
+ return betaBest;
+ }
+
+ private static int[] getThreshold(int block_size) {
+ switch (block_size) {
+ case 32:
+ return new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ case 64:
+ return new int[] {2, 3, 5, 9, 13, 17, 19, 24, 29, 32, 33, 33, 35, 37, 39, 40, 42, 43, 44, 45, 46, 47, 48, 48, 49, 50, 50, 51, 51, 52, 52, 52};
+ case 128:
+ return new int[] {2, 3, 5, 9, 17, 22, 33, 33, 43, 52, 59, 64, 65, 65, 69, 72, 76, 79, 81, 84, 86, 88, 90, 91, 93, 94, 95, 96, 98, 99, 100, 100};
+ case 256:
+ return new int[] {2, 3, 5, 9, 17, 33, 37, 64, 65, 77, 94, 107, 119, 128, 129, 129, 136, 143, 149, 154, 159, 163, 167, 171, 175, 178, 181, 183, 186, 188, 190, 192};
+ case 512:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 65, 114, 129, 140, 171, 197, 220, 239, 256, 257, 257, 270, 282, 293, 303, 312, 320, 328, 335, 342, 348, 354, 359, 364, 368};
+ case 1024:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 128, 129, 205, 257, 257, 316, 366, 410, 448, 482, 512, 513, 513, 537, 559, 579, 598, 615, 631, 645, 659, 671, 683, 694, 704};
+ case 2048:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 228, 257, 373, 512, 513, 586, 683, 768, 844, 911, 971, 1024, 1025, 1025, 1069, 1110, 1147, 1182, 1214, 1244, 1272, 1298, 1322, 1344};
+ case 4096:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 410, 513, 683, 946, 1025, 1093, 1280, 1446, 1593, 1725, 1844, 1951, 2048, 2049, 2049, 2130, 2206, 2276, 2341, 2402, 2458, 2511, 2560};
+ case 8192:
+ return new int[] {2, 3, 5, 9, 17, 33, 65, 129, 257, 513, 745, 1025, 1261, 1756, 2049, 2049, 2410, 2731, 3019, 3277, 3511, 3724, 3918, 4096, 4097, 4097, 4248, 4389, 4520, 4643, 4757, 4864};
+ default:
+ return new int[] {2, 3, 5, 8, 9, 11, 14, 16, 17, 17, 18, 19, 20, 21, 22, 22, 23, 24, 24, 24, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27};
+ }
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int l, int block_size, int[] encodingType) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int k : list) {
+ if (k > maxValue) {
+ maxValue = k;
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int[] bitWidthList = new int[l];
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(l, encode_pos, encoded_result);
+ encode_pos += 1;
+ for (int i = 0; i < l; i++) {
+ intByte2Bytes(beta[i], encode_pos + i, encoded_result);
+ }
+ encode_pos += l;
+
+ int bw = bitWidth(block_size);
+ int shiftSoFar = 0;
+ for (int i = 0; i < l; i++) {
+ int mask = (1 << beta[i]) - 1;
+ int maxValuePart = 0;
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftSoFar) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ shiftSoFar += beta[i];
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int preTypePos = encode_pos;
+ encode_pos += (l + 3) / 4;
+
+ for (int i = 0; i < l; i++) {
+
+ if (encodingType[i] == 2) {
+
+ Set<Integer> uniqueValues = new HashSet<>();
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ uniqueValues.add(currentNumber);
+ }
+ int cardinality = uniqueValues.size();
+
+ int dict_bit_width = bitWidth(cardinality) ;
+
+ List<Integer> sortedUnique = new ArrayList<>(uniqueValues);
+ Collections.sort(sortedUnique);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ int[] dict_key_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(sortedUnique.get(j), j);
+ dict_key_list[j] = sortedUnique.get(j);
+
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ encoded_result[encode_pos] = (byte) (cardinality >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (cardinality & 0xFF);
+ encode_pos += 1;
+
+ encode_pos = bitPacking(dict_key_list, bitWidthList[i], encode_pos, encoded_result, cardinality);
+
+ encode_pos = bitPacking(subcolumnList[i], dict_bit_width, encode_pos, encoded_result, list_length);
+ continue;
+ }
+
+ if (encodingType[i] == 0) {
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ }
+
+ index++;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+
+ }
+
+ preTypePos = bitPacking(encodingType, 2, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int l = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+ int[] beta = new int[l];
+ for (int i = 0; i < l; i++) {
+ beta[i] = bytes2Integer(encoded_result, encode_pos + i, 1);
+ }
+ encode_pos += l;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 2, l, encodingType);
+
+ for (int i = 0; i < l; i++) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else if(type == 1) {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }else {
+ int cardinality = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+ encode_pos += 2;
+ int dict_bit_width = bitWidth(cardinality);
+ int[] dict_key_list = new int[cardinality];
+ int[] dict_value_list = new int[cardinality];
+
+ for (int j = 0; j < cardinality; j++) {
+ dict_value_list[j] = j;
+ }
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidthList[i], cardinality, dict_key_list);
+ // encode_pos = decodeBitPacking(encoded_result, encode_pos, dict_bit_width, cardinality, dict_value_list);
+
+ encode_pos =decodeBitPacking(encoded_result, encode_pos, dict_bit_width, list_length, subcolumnList[i]);
+ Map<Integer, Integer> valueToCode = new HashMap<>();
+ for (int j = 0; j < cardinality; j++) {
+ valueToCode.put(dict_value_list[j], dict_key_list[j]);
+ }
+
+ for (int j = 0; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ int encodedValue = valueToCode.get(currentNumber);
+ subcolumnList[i][j] = encodedValue;
+ }
+
+ }
+ }
+
+ int shiftSoFar = 0;
+ for (int i = 0; i < l; i++) {
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftSoFar;
+ }
+ shiftSoFar += beta[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ private static final int TEMP_ENCODE_BUF_SIZE = 256 * 1024;
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ if (m == 0) {
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos, encoded_result,
+ beta, 1, block_size, new int[] {0});
+ return encode_pos;
+ }
+
+ int[] encodingTypeVar = new int[Math.max(m, 1)];
+ int[] betaOut = new int[Math.max(m, 1)];
+ int lVar = Subcolumn(data_delta, remainder, m, block_size, encodingTypeVar, betaOut);
+
+ byte[] tempVar = new byte[TEMP_ENCODE_BUF_SIZE];
+ int posVar = SubcolumnEncoder(data_delta, 0, tempVar, betaOut, lVar, block_size, encodingTypeVar);
+ int sizeVar = posVar;
+
+ int[] encodingTypeFixed = new int[Math.max(m, 1)];
+ int betaFixed = SubcolumnFixed(data_delta, remainder, m, block_size, encodingTypeFixed);
+ int lFixed = (m + betaFixed - 1) / betaFixed;
+ int[] betaFixedArr = new int[33];
+ for (int i = 0; i < lFixed; i++) {
+ betaFixedArr[i] = betaFixed;
+ }
+
+ byte[] tempFixed = new byte[TEMP_ENCODE_BUF_SIZE];
+ int posFixed = SubcolumnEncoder(data_delta, 0, tempFixed, betaFixedArr, lFixed, block_size, encodingTypeFixed);
+ int sizeFixed = posFixed;
+
+ if (sizeVar <= sizeFixed) {
+ System.arraycopy(tempVar, 0, encoded_result, encode_pos, sizeVar);
+ encode_pos += sizeVar;
+ } else {
+ System.arraycopy(tempFixed, 0, encoded_result, encode_pos, sizeFixed);
+ encode_pos += sizeFixed;
+ }
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[33];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "subcolumn_variable_alpha_sign_exp_mass.csv";
+
+ int repeatTime = 50;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+ while (loader.readRecord()) {
+ String v = loader.getValues()[0];
+ if (v == null || v.isEmpty()) {
+ continue;
+ }
+ data1.add(Double.valueOf(v));
+ }
+ inputStream.close();
+
+ double[] data = new double[data1.size()];
+ for (int i = 0; i < data1.size(); i++) {
+ data[i] = data1.get(i);
+ }
+
+ byte[] encoded_result = new byte[Math.max(1024, data.length * 32)];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = encodeIeee754Columns(data, data.length, 0, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ decodeIeee754Columns(encoded_result, 0);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ double[] data2_arr_decoded = decodeIeee754Columns(encoded_result, 0);
+ for (int i = 0; i < data.length; i++) {
+ // compare raw bits to avoid NaN normalization issues
+ long a = Double.doubleToRawLongBits(data[i]);
+ long b = Double.doubleToRawLongBits(data2_arr_decoded[i]);
+ if (a != b) {
+ throw new AssertionError("Mismatch at " + i + ": " + data[i] + " vs " + data2_arr_decoded[i]);
+ }
+ }
+
+ String[] record = {
+ datasetName,
+ "Sub-columns (IEEE754 Sign/Exp/Mantissa, per-column best)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutBPETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutBPETest.java
new file mode 100644
index 0000000..f3f7810
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutBPETest.java
@@ -0,0 +1,16 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import org.apache.iotdb.tsfile.encoding.SubcolumnAblationPruneNewEngine.Mode;
+
+public class SubcolumnWithoutBPETest {
+
+ @Test
+ public void test0() throws IOException {
+ SubcolumnAblationPruneNewEngine.runAblationBenchmark(
+ "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_without_bp.csv", Mode.WITHOUT_BPE);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutDETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutDETest.java
new file mode 100644
index 0000000..d0c00d8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutDETest.java
@@ -0,0 +1,16 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import org.apache.iotdb.tsfile.encoding.SubcolumnAblationPruneNewEngine.Mode;
+
+public class SubcolumnWithoutDETest {
+
+ @Test
+ public void test0() throws IOException {
+ SubcolumnAblationPruneNewEngine.runAblationBenchmark(
+ "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_without_de.csv", Mode.WITHOUT_DE);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutRLETest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutRLETest.java
new file mode 100644
index 0000000..62fb33a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SubcolumnWithoutRLETest.java
@@ -0,0 +1,16 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import org.apache.iotdb.tsfile.encoding.SubcolumnAblationPruneNewEngine.Mode;
+
+public class SubcolumnWithoutRLETest {
+
+ @Test
+ public void test0() throws IOException {
+ SubcolumnAblationPruneNewEngine.runAblationBenchmark(
+ "/Users/xiaojinzhao/Documents/GitHub/subcolumn/result/subcolumn_without_rle.csv", Mode.WITHOUT_RLE);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFLongTest.java
new file mode 100644
index 0000000..bf5c5a3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFLongTest.java
@@ -0,0 +1,694 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
+import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+public class TSDIFFLongTest {
+
+ public static long combine2Int(int int1, int int2) {
+ return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
+ }
+
+ public static int getTime(long long1) {
+ return ((int) (long1 >> 32));
+ }
+
+ public static int getValue(long long1) {
+ return ((int) (long1));
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int getBitWith(int num) {
+ if (num == 0)
+ return 1;
+ else
+ return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static int getBitWith(long num) {
+ if (num == 0)
+ return 1;
+ else
+ return 64 - Long.numberOfLeadingZeros(num);
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ int buffer = 0;
+ int leftSize = 32;
+
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+ }
+
+ public static void pack8ValuesLong(ArrayList<Long> values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ long buffer = 0;
+ int leftSize = 64;
+
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ for (int j = 0; j < 8; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((7 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static void unpack8ValuesLong(byte[] encoded, int offset, int width, ArrayList<Long> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add((buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static int bitPackingLong(ArrayList<Long> numbers, int start, int bit_width, int encode_pos,
+ byte[] encoded_result) {
+ int block_num = (numbers.size() - start) / 8;
+ for (int i = 0; i < block_num; i++) {
+ pack8ValuesLong(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8Values(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+
+ }
+ return result_list;
+ }
+
+ public static ArrayList<Long> decodeBitPackingLong(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Long> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) {
+ unpack8ValuesLong(encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size + 1;
+ int end = i * block_size + remaining;
+
+ long tmp_j_1 = ts_block[base - 1];
+ min_delta[0] = tmp_j_1;
+ int j = base;
+ long tmp_j;
+
+ while (j < end) {
+ tmp_j = ts_block[j];
+ long epsilon_v = tmp_j - tmp_j_1;
+ ts_block_delta[j - base] = epsilon_v;
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ tmp_j_1 = tmp_j;
+ j++;
+ }
+ j = 0;
+ end = remaining - 1;
+ while (j < end) {
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+ j++;
+ }
+
+ min_delta[1] = value_delta_min;
+ min_delta[2] = (value_delta_max - value_delta_min);
+
+ return ts_block_delta;
+ }
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Long> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPackingLong(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0;
+ int cur_number_bits = 0;
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width;
+
+ if (cur_number_bits + bit_width >= 64) {
+ cur_remaining <<= (64 - cur_number_bits);
+ cur_bit_width = bit_width - 64 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ // long2intBytes(cur_remaining, encode_pos, encoded_result);
+ // encode_pos += 4;
+ long2Bytes(cur_remaining, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ // cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
+ cur_remaining += (((cur_value << (64 - cur_bit_width)) & 0xFFFFFFFFFFFFFFFFL) >> (64 - cur_bit_width));
+ }
+ cur_remaining <<= (64 - cur_number_bits);
+ long2Bytes(cur_remaining, encode_pos, encoded_result);
+ encode_pos += 8;
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Long> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Long> result_list = new ArrayList<>(
+ decodeBitPackingLong(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ // int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ // decode_pos += 4;
+ int_remaining.add(bytes2Long(encoded, decode_pos, 8));
+ decode_pos += 8;
+ }
+
+ int cur_remaining_bits = 64;
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ long tmp = (long) (cur_number >> (64 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ long tmp = (long) (cur_number >> (64 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (64 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFFFFFFFFFL;
+ cur_remaining_bits = 64 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static int BOSBlockEncoder(long[] ts_block, int block_i, int block_size, int remaining, int encode_pos,
+ byte[] cur_byte) {
+
+ long[] min_delta = new long[3];
+ long[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ long2Bytes(min_delta[0], encode_pos, cur_byte);
+ encode_pos += 8;
+ long2Bytes(min_delta[1], encode_pos, cur_byte);
+ encode_pos += 8;
+
+ int bit_width_final = getBitWith(min_delta[2]);
+ intByte2Bytes(bit_width_final, encode_pos, cur_byte);
+ encode_pos += 1;
+ ArrayList<Long> final_normal = new ArrayList<>();
+ for (long value : ts_block_delta) {
+ final_normal.add(value);
+ }
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final, encode_pos, cur_byte);
+ return encode_pos;
+ }
+
+ public static int BOSEncoder(
+ long[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ for (int i = 0; i < block_num; i++) {
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ long2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all - start;
+ encode_pos = BOSBlockEncoder(data, block_num, block_size, remaining, encode_pos, encoded_result);
+
+ }
+
+ return encode_pos;
+ }
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, long[] value_list, int block_size,
+ int[] value_pos_arr) {
+
+ long value0 = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value0;
+ value_pos_arr[0]++;
+
+ long min_delta = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ ArrayList<Long> final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size,
+ decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ int normal_i = 0;
+ long pre_v = value0;
+
+ for (int i = 0; i < block_size; i++) {
+ long current_delta = min_delta + final_normal.get(normal_i);
+ pre_v = current_delta + pre_v;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ long[] value_list = new long[length_all + block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size, value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ long value_end = bytes2Long(encoded, decode_pos, 8);
+ decode_pos += 8;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length--;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "ts2diff_long.csv";
+
+ int block_size = 1024;
+
+ int repeatTime = 500;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ // f = tempList[1];
+ // System.out.println(f);
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = BOSEncoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ BOSDecoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "TS2DIFF",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+ writer.close();
+
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnLongTest.java
new file mode 100644
index 0000000..6acc9e5
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnLongTest.java
@@ -0,0 +1,366 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class TSDIFFSubcolumnLongTest {
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Encoder(long[] data, int block_size, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ encoded_result[0] = (byte) (data_length >> 24);
+ encoded_result[1] = (byte) (data_length >> 16);
+ encoded_result[2] = (byte) (data_length >> 8);
+ encoded_result[3] = (byte) data_length;
+ encode_pos += 4;
+
+ encoded_result[4] = (byte) (block_size >> 24);
+ encoded_result[5] = (byte) (block_size >> 16);
+ encoded_result[6] = (byte) (block_size >> 8);
+ encoded_result[7] = (byte) block_size;
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ int[] beta = new int[1];
+ beta[0] = 3;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ long value = data[num_blocks * block_size + i];
+ long2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16)
+ |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int block_size = ((encoded_result[encode_pos] & 0xFF) << 24) | ((encoded_result[encode_pos + 1] & 0xFF) << 16) |
+ ((encoded_result[encode_pos + 2] & 0xFF) << 8) | (encoded_result[encode_pos + 3] & 0xFF);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+ public static long[] getAbsDeltaTsBlock(
+ long[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ long[] min_delta) {
+ long[] ts_block_delta = new long[remaining - 1];
+
+ long value_delta_min = Long.MAX_VALUE;
+ long value_delta_max = Long.MIN_VALUE;
+ int base = i * block_size + 1;
+ int end = i * block_size + remaining;
+
+ long tmp_j_1 = ts_block[base - 1];
+ min_delta[0] = tmp_j_1;
+ int j = base;
+ long tmp_j;
+
+ while (j < end) {
+ tmp_j = ts_block[j];
+ long epsilon_v = tmp_j - tmp_j_1;
+ ts_block_delta[j - base] = epsilon_v;
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ tmp_j_1 = tmp_j;
+ j++;
+ }
+ j = 0;
+ end = remaining - 1;
+ while (j < end) {
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+ j++;
+ }
+
+ min_delta[1] = value_delta_min;
+ min_delta[2] = (value_delta_max - value_delta_min);
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ long[] min_delta = new long[3];
+
+ long[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size, remainder, min_delta);
+
+ long2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ long2Bytes(min_delta[1], encode_pos, encoded_result);
+ encode_pos += 8;
+
+ if (block_index == 0) {
+ long maxValue = 0;
+ for (int j = 0; j < remainder - 1; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = SubcolumnLongTest.bitWidth(maxValue);
+
+ beta[0] = SubcolumnLongTest.Subcolumn(data_delta, remainder - 1, m, block_size);
+ }
+
+ encode_pos = SubcolumnLongTest.SubcolumnEncoder(data_delta, encode_pos, encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, long[] data) {
+ long[] min_delta = new long[3];
+
+ min_delta[0] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ min_delta[1] = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ long[] data_delta = new long[remainder - 1];
+
+ encode_pos = SubcolumnLongTest.SubcolumnDecoder(encoded_result, encode_pos, data_delta, block_size);
+
+ for (int i = 0; i < remainder - 1; i++) {
+ data_delta[i] = data_delta[i] + min_delta[1];
+ }
+
+ data[block_index * block_size] = min_delta[0];
+
+ for (int i = 0; i < remainder - 1; i++) {
+ data[block_index * block_size + i + 1] = data[block_index * block_size + i] + data_delta[i];
+ }
+
+ return encode_pos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "ts2diff_subcolumn_long.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 500;
+
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "TS2DIFF+Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnPruneNewTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnPruneNewTest.java
new file mode 100644
index 0000000..1f626a7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnPruneNewTest.java
@@ -0,0 +1,396 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class TSDIFFSubcolumnPruneNewTest {
+
+ public static int Encoder(int[] data, int blockSize, byte[] encodedResult) {
+ return Encoder(data, blockSize, encodedResult, null, null);
+ }
+
+ public static int Encoder(
+ int[] data, int blockSize, byte[] encodedResult, long[] ts2diffTime, long[] subcolumnTime) {
+ int dataLength = data.length;
+ int encodePos = 0;
+
+ encodedResult[0] = (byte) (dataLength >> 24);
+ encodedResult[1] = (byte) (dataLength >> 16);
+ encodedResult[2] = (byte) (dataLength >> 8);
+ encodedResult[3] = (byte) dataLength;
+ encodePos += 4;
+
+ encodedResult[4] = (byte) (blockSize >> 24);
+ encodedResult[5] = (byte) (blockSize >> 16);
+ encodedResult[6] = (byte) (blockSize >> 8);
+ encodedResult[7] = (byte) blockSize;
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int[] beta = new int[] {3};
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos =
+ BlockEncoder(
+ data, i, blockSize, blockSize, encodePos, encodedResult, beta, ts2diffTime, subcolumnTime);
+ }
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[numBlocks * blockSize + i];
+ encodedResult[encodePos] = (byte) (value >> 24);
+ encodedResult[encodePos + 1] = (byte) (value >> 16);
+ encodedResult[encodePos + 2] = (byte) (value >> 8);
+ encodedResult[encodePos + 3] = (byte) value;
+ encodePos += 4;
+ }
+ } else {
+ encodePos =
+ BlockEncoder(
+ data, numBlocks, blockSize, remainder, encodePos, encodedResult, beta, ts2diffTime, subcolumnTime);
+ }
+
+ return encodePos;
+ }
+
+ public static int[] Decoder(byte[] encodedResult) {
+ int encodePos = 0;
+
+ int dataLength =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int blockSize =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int numBlocks = dataLength / blockSize;
+ int[] data = new int[dataLength];
+
+ for (int i = 0; i < numBlocks; i++) {
+ encodePos = BlockDecoder(encodedResult, i, blockSize, blockSize, encodePos, data);
+ }
+
+ int remainder = dataLength % blockSize;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[numBlocks * blockSize + i] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+ }
+ } else {
+ encodePos = BlockDecoder(encodedResult, numBlocks, blockSize, remainder, encodePos, data);
+ }
+
+ return data;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] tsBlock, int blockIndex, int blockSize, int remaining, int[] minDelta) {
+ int[] tsBlockDelta = new int[remaining - 1];
+ fillAbsDeltaTsBlock(tsBlock, blockIndex, blockSize, remaining, minDelta, tsBlockDelta);
+ return tsBlockDelta;
+ }
+
+ private static void fillAbsDeltaTsBlock(
+ int[] tsBlock,
+ int blockIndex,
+ int blockSize,
+ int remaining,
+ int[] minDelta,
+ int[] tsBlockDelta) {
+ int valueDeltaMin = Integer.MAX_VALUE;
+ int valueDeltaMax = Integer.MIN_VALUE;
+ int base = blockIndex * blockSize + 1;
+ int end = blockIndex * blockSize + remaining;
+
+ int prev = tsBlock[base - 1];
+ minDelta[0] = prev;
+ int j = base;
+ while (j < end) {
+ int cur = tsBlock[j];
+ int epsilon = cur - prev;
+ tsBlockDelta[j - base] = epsilon;
+ if (epsilon < valueDeltaMin) {
+ valueDeltaMin = epsilon;
+ }
+ if (epsilon > valueDeltaMax) {
+ valueDeltaMax = epsilon;
+ }
+ prev = cur;
+ j++;
+ }
+
+ for (j = 0; j < remaining - 1; j++) {
+ tsBlockDelta[j] = tsBlockDelta[j] - valueDeltaMin;
+ }
+
+ minDelta[1] = valueDeltaMin;
+ minDelta[2] = valueDeltaMax - valueDeltaMin;
+ }
+
+ public static int BlockEncoder(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta) {
+ return BlockEncoder(
+ data, blockIndex, blockSize, remainder, encodePos, encodedResult, beta, null, null);
+ }
+
+ public static int BlockEncoder(
+ int[] data,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ byte[] encodedResult,
+ int[] beta,
+ long[] ts2diffTime,
+ long[] subcolumnTime) {
+ long ts2diffStart = System.nanoTime();
+ int[] minDelta = SubcolumnPruneNewTest.borrowMinDelta3Buffer();
+ int[] dataDelta = SubcolumnPruneNewTest.borrowDataDeltaBuffer();
+ fillAbsDeltaTsBlock(data, blockIndex, blockSize, remainder, minDelta, dataDelta);
+
+ encodedResult[encodePos] = (byte) (minDelta[0] >> 24);
+ encodedResult[encodePos + 1] = (byte) (minDelta[0] >> 16);
+ encodedResult[encodePos + 2] = (byte) (minDelta[0] >> 8);
+ encodedResult[encodePos + 3] = (byte) minDelta[0];
+ encodePos += 4;
+
+ encodedResult[encodePos] = (byte) (minDelta[1] >> 24);
+ encodedResult[encodePos + 1] = (byte) (minDelta[1] >> 16);
+ encodedResult[encodePos + 2] = (byte) (minDelta[1] >> 8);
+ encodedResult[encodePos + 3] = (byte) minDelta[1];
+ encodePos += 4;
+
+ int maxValue = 0;
+ for (int v : dataDelta) {
+ if (v > maxValue) {
+ maxValue = v;
+ }
+ }
+ int m = SubcolumnPruneNewTest.bitWidth(maxValue);
+ int[] encodingType = SubcolumnPruneNewTest.borrowEncodingTypeBuffer();
+ long ts2diffEnd = System.nanoTime();
+ if (ts2diffTime != null) {
+ ts2diffTime[0] += (ts2diffEnd - ts2diffStart);
+ }
+
+ long subStart = System.nanoTime();
+ beta[0] =
+ SubcolumnPruneNewTest.Subcolumn(dataDelta, remainder - 1, m, blockSize, encodingType);
+ encodePos =
+ SubcolumnPruneNewTest.SubcolumnEncoder(
+ dataDelta,
+ remainder - 1,
+ encodePos,
+ encodedResult,
+ beta,
+ blockSize,
+ encodingType,
+ m);
+ long subEnd = System.nanoTime();
+ if (subcolumnTime != null) {
+ subcolumnTime[0] += (subEnd - subStart);
+ }
+
+ return encodePos;
+ }
+
+ public static int BlockDecoder(
+ byte[] encodedResult,
+ int blockIndex,
+ int blockSize,
+ int remainder,
+ int encodePos,
+ int[] data) {
+ int[] minDelta = new int[3];
+
+ minDelta[0] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ minDelta[1] =
+ ((encodedResult[encodePos] & 0xFF) << 24)
+ | ((encodedResult[encodePos + 1] & 0xFF) << 16)
+ | ((encodedResult[encodePos + 2] & 0xFF) << 8)
+ | (encodedResult[encodePos + 3] & 0xFF);
+ encodePos += 4;
+
+ int[] dataDelta = new int[remainder - 1];
+ encodePos = SubcolumnPruneNewTest.SubcolumnDecoder(encodedResult, encodePos, dataDelta, blockSize);
+
+ for (int i = 0; i < remainder - 1; i++) {
+ dataDelta[i] = dataDelta[i] + minDelta[1];
+ }
+
+ data[blockIndex * blockSize] = minDelta[0];
+ for (int i = 0; i < remainder - 1; i++) {
+ data[blockIndex * blockSize + i + 1] = data[blockIndex * blockSize + i] + dataDelta[i];
+ }
+ return encodePos;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+ if (decimalIndex == -1) {
+ return 0;
+ }
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+ File file = new File(path);
+ String fileName = file.getName();
+ int dotIndex = fileName.lastIndexOf('.');
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ // String parentDir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ String parentDir = "D:/github/xjz17/subcolumn/";
+
+ String inputParentDir = parentDir + "dataset/";
+
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "ts2diff_subcolumn_adddict_prunenew_opt2.csv";
+
+ int blockSize = 512;
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio",
+ "TS2DIFF Time",
+ "Subcolumn Encode Time"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data1.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int[] data2Arr = new int[data1.size()];
+ int maxMul = (int) Math.pow(10, maxDecimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2Arr[i] = (int) (data1.get(i) * maxMul);
+ }
+
+ byte[] encodedResult = new byte[data2Arr.length * 8];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ long ts2diffTime = 0;
+ long subcolumnEncodeTime = 0;
+ double compressedSize = 0;
+ int length = 0;
+ long[] ts2diffTimeArr = new long[1];
+ long[] subcolumnEncodeTimeArr = new long[1];
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ ts2diffTimeArr[0] = 0;
+ subcolumnEncodeTimeArr[0] = 0;
+ length =
+ Encoder(
+ data2Arr, blockSize, encodedResult, ts2diffTimeArr, subcolumnEncodeTimeArr);
+ ts2diffTime += ts2diffTimeArr[0];
+ subcolumnEncodeTime += subcolumnEncodeTimeArr[0];
+ }
+ long e = System.nanoTime();
+ encodeTime += (e - s) / repeatTime;
+ ts2diffTime /= repeatTime;
+ subcolumnEncodeTime /= repeatTime;
+ compressedSize += length;
+
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ Decoder(encodedResult);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s) / repeatTime;
+
+ double ratio = compressedSize / (double) (Math.max(1, data1.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "TS2DIFF+Sub-columns(AddDictPruneNew-Opt2)",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressedSize),
+ String.valueOf(ratio),
+ String.valueOf(ts2diffTime),
+ String.valueOf(subcolumnEncodeTime)
+ });
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnTest.java
index 11092b8..810e0f4 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFSubcolumnTest.java
@@ -149,7 +149,6 @@
int encode_pos, byte[] encoded_result, int[] beta) {
int[] min_delta = new int[3];
- // data_delta 的长度为 remainder - 1
int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size, remainder, min_delta);
encoded_result[encode_pos] = (byte) (min_delta[0] >> 24);
@@ -211,15 +210,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -241,24 +237,19 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
- String outputPath = output_parent_dir + "ts2diff_subcolumn.csv";
+ String outputPath = output_parent_dir + "ts2diff_subcolumn2.csv";
int block_size = 512;
- int repeatTime = 200;
+ int repeatTime = 500;
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -275,7 +266,6 @@
writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -328,11 +318,7 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
@@ -363,133 +349,4 @@
writer.close();
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "ts2diff_subcolumn.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = Encoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- int[] data2_arr_decoded = new int[data1.size()];
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- data2_arr_decoded = Decoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "TS2DIFF+Sub-columns",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFTest.java
index cd25f03..ffeab23 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFTest.java
@@ -19,10 +19,10 @@
import java.util.List;
import java.util.stream.Stream;
-import static java.lang.Math.pow;
-
public class TSDIFFTest {
+ private static final int BIT_IO_STEP = 2;
+
public static long combine2Int(int int1, int int2) {
return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
}
@@ -96,16 +96,12 @@
byte[] encoded_result) {
int bufIdx = 0;
int valueIdx = offset;
- // remaining bits for the current unfinished Integer
int leftBit = 0;
while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
int buffer = 0;
- // remaining size of bits in the 'buffer'
int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
if (leftBit > 0) {
buffer |= (values.get(valueIdx) << (32 - leftBit));
leftSize -= leftBit;
@@ -114,20 +110,15 @@
}
while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
buffer |= (values.get(valueIdx) << (leftSize - width));
leftSize -= width;
valueIdx++;
}
- // If the remaining space of the buffer can not save the bits for one Integer,
if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
buffer |= (values.get(valueIdx) >>> (width - leftSize));
leftBit = width - leftSize;
}
- // put the buffer into the final result
for (int j = 0; j < 4; j++) {
encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
encode_pos++;
@@ -137,29 +128,21 @@
}
}
}
- // return encode_pos;
}
public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
int byteIdx = offset;
long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
int totalBits = 0;
int valueIdx = 0;
while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
while (totalBits < width) {
buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
byteIdx++;
totalBits += 8;
}
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
while (totalBits >= width && valueIdx < 8) {
result_list.add((int) (buffer >>> (totalBits - width)));
valueIdx++;
@@ -186,7 +169,7 @@
ArrayList<Integer> result_list = new ArrayList<>();
int block_num = (block_size - 1) / 8;
- for (int i = 0; i < block_num; i++) { // bitpacking
+ for (int i = 0; i < block_num; i++) {
unpack8Values(encoded, decode_pos, bit_width, result_list);
decode_pos += bit_width;
@@ -251,7 +234,7 @@
int cur_number_bits = 0; // the bit width used of encoded int
for (int i = n_k_b * 8; i < n_k; i++) {
long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
+ int cur_bit_width = bit_width;
if (cur_number_bits + bit_width >= 32) {
cur_remaining <<= (32 - cur_number_bits);
@@ -294,7 +277,7 @@
decode_pos += 4;
}
- int cur_remaining_bits = 32; // remaining bit width of current value
+ int cur_remaining_bits = 32;
long cur_number = int_remaining.get(0);
int cur_number_i = 1;
for (int i = n_k_b * 8; i < length; i++) {
@@ -384,29 +367,22 @@
int encode_pos,
byte[] cur_byte,
int[] bit_index_list) {
- // 找到要插入的位的索引
- int bit_index = bit_index_list[0];// cur_byte[encode_pos + 1];
+ int bit_index = bit_index_list[0];
- // 计算数值的起始位位置
int remaining_bits = bit_width;
while (remaining_bits > 0) {
- // 计算在当前字节中可以使用的位数
int available_bits = bit_index;
- int bits_to_write = Math.min(available_bits, remaining_bits);
+ int bits_to_write = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
- // 更新 bit_index
bit_index = available_bits - bits_to_write;
- // 计算要写入的位的掩码和数值
int mask = (1 << bits_to_write) - 1;
int bits = (num >> (remaining_bits - bits_to_write)) & mask;
- // 写入到当前位置
- cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index);
cur_byte[encode_pos] |= (byte) (bits << bit_index);
- // 更新位宽和数值
remaining_bits -= bits_to_write;
if (bit_index == 0) {
bit_index = 8;
@@ -414,7 +390,6 @@
}
}
bit_index_list[0] = bit_index;
- // cur_byte[encode_pos + 1] = (byte) bit_index;
return encode_pos;
}
@@ -431,7 +406,6 @@
int bit_width_final = getBitWith(min_delta[2]);
intByte2Bytes(bit_width_final, encode_pos, cur_byte);
encode_pos += 1;
- // ArrayList<Integer> final_normal = new ArrayList<>();
int[] bit_index_list = new int[1];
bit_index_list[0] = 8;
for (int value : ts_block_delta) {
@@ -440,8 +414,6 @@
if (bit_index_list[0] != 8) {
encode_pos++;
}
- // encode_pos = encodeOutlier2Bytes(final_normal,
- // bit_width_final,encode_pos,cur_byte);
return encode_pos;
}
@@ -547,24 +519,87 @@
}
}
+ public static int[] decodeToIntArray(byte[] encoded) {
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all + block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size, value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length--;
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ return Arrays.copyOf(value_list, length_all);
+ }
+
+ public static int[] decodeToIntArrayImprove(byte[] encoded) {
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all + block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos =
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length--;
+ decode_pos =
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ return Arrays.copyOf(value_list, length_all);
+ }
+
public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
int decode_pos = decode_pos_list[0];
- int bit_index = decode_pos_list[1]; // cur_byte[decode_pos + 1];
+ int bit_index = decode_pos_list[1];
int remaining_bits = bit_width;
int num = 0;
while (remaining_bits > 0) {
int available_bits = bit_index;
- int bits_to_read = Math.min(available_bits, remaining_bits);
+ int bits_to_read = Math.min(BIT_IO_STEP, Math.min(available_bits, remaining_bits));
- // 计算要读取的位的掩码
int mask = (1 << bits_to_read) - 1;
int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
- // 将读取的位合并到结果中
num = (num << bits_to_read) | bits;
- // 更新位宽和 bit_index
remaining_bits -= bits_to_read;
bit_index = available_bits - bits_to_read;
@@ -703,15 +738,12 @@
}
public static int getDecimalPrecision(String str) {
- // 查找小数点的位置
int decimalIndex = str.indexOf(".");
- // 如果没有小数点，精度为0
if (decimalIndex == -1) {
return 0;
}
- // 获取小数点后的部分并返回其长度
return str.substring(decimalIndex + 1).length();
}
@@ -733,24 +765,19 @@
}
@Test
- public void testSubcolumn() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
+ public void test0() throws IOException {
+ String parent_dir = "D://github/xjz17/subcolumn/";
String input_parent_dir = parent_dir + "dataset/";
- String output_parent_dir = "D:/encoding-subcolumn/result/";
- // String output_parent_dir = parent_dir + "result/";
+ String output_parent_dir = parent_dir + "result/";
String outputPath = output_parent_dir + "ts2diff.csv";
int block_size = 1024;
- int repeatTime = 100;
+ int repeatTime = 500;
- // repeatTime = 1;
-
- List<String> integerDatasets = new ArrayList<>();
- integerDatasets.add("Wine-Tasting");
CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
writer.setRecordDelimiter('\n');
@@ -764,9 +791,8 @@
"Compressed Size",
"Compression Ratio"
};
- writer.writeRecord(head); // write header to output file
+ writer.writeRecord(head);
File directory = new File(input_parent_dir);
- // File[] csvFiles = directory.listFiles();
File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
for (File file : csvFiles) {
@@ -778,9 +804,6 @@
CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
ArrayList<Float> data1 = new ArrayList<>();
- // ArrayList<Integer> data2 = new ArrayList<>();
-
- // loader.readHeaders();
int max_decimal = 0;
while (loader.readRecord()) {
@@ -792,10 +815,7 @@
if (cur_decimal > max_decimal) {
max_decimal = cur_decimal;
}
- // String value = loader.getValues()[index];
data1.add(Float.valueOf(f_str));
- // data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
}
inputStream.close();
@@ -806,7 +826,7 @@
}
System.out.println(max_decimal);
- byte[] encoded_result = new byte[data2_arr.length * 4];
+ byte[] encoded_result = new byte[data2_arr.length * 8];
long encodeTime = 0;
long decodeTime = 0;
double ratio = 0;
@@ -816,7 +836,8 @@
long s = System.nanoTime();
for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = BOSEncoder(data2_arr, block_size, encoded_result);
+ // length = BOSEncoder(data2_arr, block_size, encoded_result);
+ length = BOSEncoderImprove(data2_arr, block_size, encoded_result);
}
long e = System.nanoTime();
@@ -825,18 +846,15 @@
double ratioTmp;
- if (integerDatasets.contains(datasetName)) {
- ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- } else {
- ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
- }
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
ratio += ratioTmp;
s = System.nanoTime();
for (int repeat = 0; repeat < repeatTime; repeat++) {
- BOSDecoder(encoded_result);
+ // BOSDecoder(encoded_result);
+ BOSDecoderImprove(encoded_result);
}
e = System.nanoTime();
@@ -859,131 +877,4 @@
}
- @Test
- public void testTransData() throws IOException {
- String parent_dir = "D:/github/xjz17/subcolumn/";
-
- String output_parent_dir = "D:/encoding-subcolumn/trans_data_result/";
- // String output_parent_dir = parent_dir + "trans_data_result/";
-
- String input_parent_dir = parent_dir + "trans_data/";
-
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
-
- try (Stream<Path> paths = Files.walk(Paths.get(input_parent_dir))) {
- paths.filter(Files::isDirectory)
- .filter(path -> !path.equals(Paths.get(input_parent_dir)))
- .forEach(dir -> {
- String name = dir.getFileName().toString();
- dataset_name.add(name);
- input_path_list.add(dir.toString());
- dataset_block_size.add(1024);
- });
- }
-
- String outputPath = output_parent_dir + "ts2diff.csv";
- CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
- writer.setRecordDelimiter('\n');
-
- String[] head = {
- "Dataset",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head);
-
- int repeatTime = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- long totalEncodeTime = 0;
- long totalDecodeTime = 0;
- double totalCompressedSize = 0;
- int totalPoints = 0;
-
- for (File f : tempList) {
- String datasetName = extractFileName(f.toString());
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
- while (loader.readRecord()) {
- // String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- // data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
-
- for (int repeat = 0; repeat < repeatTime; repeat++) {
- BOSDecoder(encoded_result);
- }
-
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime);
-
- totalEncodeTime += encodeTime;
- totalDecodeTime += decodeTime;
- totalCompressedSize += compressed_size;
- totalPoints += data1.size();
-
- }
-
- double compressionRatio = totalCompressedSize / (totalPoints * Integer.BYTES);
-
- String[] record = {
- dataset_name.get(file_i),
- "TS2DIFF",
- String.valueOf(totalEncodeTime),
- String.valueOf(totalDecodeTime),
- String.valueOf(totalPoints),
- String.valueOf(totalCompressedSize),
- String.valueOf(compressionRatio)
- };
-
- writer.writeRecord(record);
- System.out.println(compressionRatio);
- }
- writer.close();
- }
-
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateAppendTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateAppendTest.java
new file mode 100644
index 0000000..c803cc3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateAppendTest.java
@@ -0,0 +1,881 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.encoding.SubcolumnTest.bytes2Integer;
+import static org.apache.iotdb.tsfile.encoding.SubcolumnTest.int2Bytes;
+import static org.apache.iotdb.tsfile.encoding.SubcolumnTest.BlockEncoder;
+
+public class UpdateAppendTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+ updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_append.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Append-only Time", // 拼接原始值的时间
+ "Append-only Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long appendCompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[1];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ int num_blocks = data_length / block_size;
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ for (int i = 0; i < remainder; i++) {
+ data2_arr[num_blocks * block_size + i] = data2_arr[num_blocks * block_size + i];
+ }
+// data2_arr[data1.size()] = updateRange.get(datasetName);
+// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int encode_pos = length;
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encode_pos = length;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data2_arr[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data2_arr, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+// length = Append(encoded_result, length, updateRange.get(datasetName) );
+// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+ }
+ length = encode_pos ;
+
+ e = System.nanoTime();
+ appendCompressTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+ String.valueOf(appendTime),
+ String.valueOf(appendCompressTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateDeleteSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateDeleteSmallerTest.java
new file mode 100644
index 0000000..66f5ad9
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateDeleteSmallerTest.java
@@ -0,0 +1,1036 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class UpdateDeleteSmallerTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+// updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_delete_smaller.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+// "Insert Time", // 拼接原始值的时间
+ "Delete Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long insertSmallerTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[3];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ if(remainder < 4){
+ continue;
+ }
+
+ int num_blocks = data_length / block_size;
+ int number_of_insert_values = block_size - remainder;
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+ Random random = new Random();
+ int randomNumber = random.nextInt(random_lower_bound);
+
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+// data2_arr[num_blocks * block_size + remainder] = randomNumber;
+//// for (int i = 0; i < number_of_insert_values; i++) {
+//// data2_arr[num_blocks * block_size + remainder + i] = remaining_values[i];
+//// }
+//// data2_arr[data1.size()] = updateRange.get(datasetName);
+//// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+// }
+
+// e = System.nanoTime();
+// appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int new_decode_posencode_pos = length;
+
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int encode_pos = length;
+
+ int beta_o = beta[0];
+
+ int new_block_encode_length = beta[2];
+ int m = bytes2Integer(encoded_result, new_block_encode_length+4, 1);
+ int bitwidth_random = bitWidth(randomNumber);
+
+ int mask = (1 << beta[0]) - 1;
+
+ int l = (m + beta_o - 1) / beta_o;
+
+ int[] bitWidthList = new int[l];
+// int[] bit_width_add_sub_columns = new int[l];
+
+ int new_decode_pos = decodeBitPacking(encoded_result, new_block_encode_length+6,
+ 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][remainder];
+ int[][] new_subcolumnList = new int[l][remainder-1];
+
+ int[] encodingType = new int[l];
+
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, 1, l, encodingType);
+
+
+ int bw = bitWidth(block_size);
+ byte[] new_encoded_result = new byte[(remainder+1)*16];
+ int pos_of_new_encoded_result = 0;
+
+ for(int j = l-1;j>=0;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ // if bit-packing
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, remainder,
+ subcolumnList[j]);
+
+ System.arraycopy(subcolumnList[j], 0, new_subcolumnList[j], 0, remainder-1);
+// new_subcolumnList[j][remainder] = add_sub_columns[j];
+ int maxValue = 0;
+ for(int i = 0;i<remainder-1;i++){
+ if (new_subcolumnList[j][i] > maxValue) {
+ maxValue = new_subcolumnList[j][i];
+ }
+ }
+ int new_bit_width = bitWidth(maxValue);
+ pos_of_new_encoded_result = bitPacking(new_subcolumnList[j], new_bit_width,
+ pos_of_new_encoded_result, new_encoded_result, remainder-1);
+ }
+ else {
+// System.out.println(encoded_result[new_decode_pos]);
+// System.out.println(encoded_result[new_decode_pos+1]);
+ // if rle
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+
+ new_decode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bw, index, run_length);
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, index, rle_values);
+
+// System.out.println(Arrays.toString(run_length));
+// System.out.println(Arrays.toString(rle_values));
+ if(run_length[index-1] != 1){
+ run_length[index-1] -= 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+ pos_of_new_encoded_result = bitPacking(rle_values, bitWidth, pos_of_new_encoded_result, new_encoded_result, index);
+
+ }else{
+ int[] new_run_length = new int[index-1];
+ int[] new_rle_values = new int[index-1];
+ System.arraycopy(new_run_length, 0, run_length, 0, index-1);
+ System.arraycopy(new_rle_values, 0, rle_values, 0, index-1);
+// new_run_length[index] = 1;
+// new_rle_values[index] = add_sub_columns[j];
+
+ index --;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+ int maxValue = 0;
+ for(int i = 0;i<index-1;i++){
+ if (new_rle_values[i] > maxValue) {
+ maxValue = new_rle_values[i];
+ }
+ }
+ int new_bit_width = bitWidth(maxValue);
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(new_run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+
+ pos_of_new_encoded_result = bitPacking(new_rle_values, new_bit_width, pos_of_new_encoded_result, new_encoded_result, index);
+
+
+ }
+
+ }
+ }
+// for (int j=l; j<add_number_of_sub_column; j++){
+// new_encodingType[j]=1;
+// int[] run_length = {remainder,1};
+// int[] rle_values = {0,add_sub_columns[j]};
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (2 >> 8);
+// pos_of_new_encoded_result += 1;
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (2 & 0xFF);
+// pos_of_new_encoded_result += 1;
+// pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, 2);
+// pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, 2);
+// }
+
+
+
+
+
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertSmallerTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertSmallerTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertLargerTest.java
new file mode 100644
index 0000000..11889c7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertLargerTest.java
@@ -0,0 +1,1045 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.*;
+
+import static java.lang.Math.floor;
+
+public class UpdateInsertLargerTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+ updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 1;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_insert_larger.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+// "Insert Time", // 拼接原始值的时间
+ "Insert Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long insertGreaterTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[3];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Update");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ if(remainder < 4){
+ continue;
+ }
+
+ int num_blocks = data_length / block_size;
+ int number_of_insert_values = block_size - remainder;
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+ Random random = new Random();
+ int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+// data2_arr[num_blocks * block_size + remainder] = randomNumber;
+//// for (int i = 0; i < number_of_insert_values; i++) {
+//// data2_arr[num_blocks * block_size + remainder + i] = remaining_values[i];
+//// }
+//// data2_arr[data1.size()] = updateRange.get(datasetName);
+//// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+// }
+
+// e = System.nanoTime();
+// appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int new_decode_posencode_pos = length;
+
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int encode_pos = length;
+
+ int beta_o = beta[0];
+
+ int new_block_encode_length = beta[2];
+ int m = bytes2Integer(encoded_result, new_block_encode_length+4, 1);
+ int bitwidth_random = bitWidth(randomNumber);
+ if(bitwidth_random>m*beta_o){
+ System.out.println("inserted");
+ // 计算要加几个sub-column
+ int number_of_sub_column_random = (bitwidth_random + beta_o - 1) / beta_o;
+ int add_number_of_sub_column = number_of_sub_column_random - m;
+ int mask = (1 << beta[0]) - 1;
+
+
+
+ int l = (m + beta_o - 1) / beta_o;
+
+ int[] bitWidthList = new int[l];
+
+ int new_decode_pos = decodeBitPacking(encoded_result, new_block_encode_length+6,
+ 8, l, bitWidthList);
+
+ int[] add_sub_columns = new int[number_of_sub_column_random];
+ int[] bit_width_add_sub_columns = new int[number_of_sub_column_random];
+// boolean[] is_changed = new boolean[number_of_sub_column_random];
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta[0];
+ add_sub_columns[i] = (randomNumber >> shiftAmount) & mask;
+ int tmp_bit_width = bitWidth(add_sub_columns[i]);
+ bit_width_add_sub_columns[i] = Math.max(tmp_bit_width, bitWidthList[i]);
+// is_changed[i]= tmp_bit_width > bitWidthList[i];
+ }
+ for (int i=l; i<add_number_of_sub_column; i++){
+ bit_width_add_sub_columns[i] = bitWidth(add_sub_columns[i]);
+ }
+
+ int[][] subcolumnList = new int[l][remainder];
+ int[][] new_subcolumnList = new int[l][remainder+1];
+
+ int[] encodingType = new int[l];
+ int[] new_encodingType = new int[number_of_sub_column_random];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, 1, l, encodingType);
+ System.arraycopy(encodingType, 0, new_encodingType, 0, l);
+// System.out.println(Arrays.toString(encodingType));
+
+ int bw = bitWidth(block_size);
+ byte[] new_encoded_result = new byte[(remainder+1)*16];
+ int pos_of_new_encoded_result = 0;
+
+ for(int j = l-1;j>=0;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ // if bit-packing
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, remainder,
+ subcolumnList[j]);
+
+// if(is_changed[j]){
+//
+//
+// }else{
+ System.arraycopy(subcolumnList[j], 0, new_subcolumnList[j], 0, remainder);
+ new_subcolumnList[j][remainder] = add_sub_columns[j];
+ pos_of_new_encoded_result = bitPacking(new_subcolumnList[j], bit_width_add_sub_columns[j],
+ pos_of_new_encoded_result, new_encoded_result, remainder+1);
+// }
+ }
+ else {
+// System.out.println(encoded_result[new_decode_pos]);
+// System.out.println(encoded_result[new_decode_pos+1]);
+ // if rle
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+
+ new_decode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bw, index, run_length);
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, index, rle_values);
+
+// System.out.println(Arrays.toString(run_length));
+// System.out.println(Arrays.toString(rle_values));
+ if(rle_values[index-1] == add_sub_columns[j]){
+ run_length[index-1] += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+ pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+
+ }else{
+ int[] new_run_length = new int[index+1];
+ int[] new_rle_values = new int[index+1];
+ System.arraycopy(new_run_length, 0, run_length, 0, index);
+ System.arraycopy(new_rle_values, 0, rle_values, 0, index);
+ new_run_length[index] = 1;
+ new_rle_values[index] = add_sub_columns[j];
+
+ index ++;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(new_run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+
+ pos_of_new_encoded_result = bitPacking(new_rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+
+
+ }
+
+ }
+ }
+ for (int j=l; j<add_number_of_sub_column; j++){
+ new_encodingType[j]=1;
+ int[] run_length = {remainder,1};
+ int[] rle_values = {0,add_sub_columns[j]};
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (2 >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (2 & 0xFF);
+ pos_of_new_encoded_result += 1;
+ pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, 2);
+ pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, 2);
+ }
+
+
+ }
+
+
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertGreaterTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertGreaterTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertOneCompressTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertOneCompressTest.java
new file mode 100644
index 0000000..d455164
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertOneCompressTest.java
@@ -0,0 +1,969 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class UpdateInsertOneCompressTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+ updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 200;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_insert_one_compress.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Compression (Larger)", // 拼接原始值的时间
+ "Insert Time with Compression (Smaller)", // 拼接原始值的时间
+// "Insert Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long insertRemainderTime = 0;
+ long insertRemainderOneTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[3];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+// if(remainder < 4){
+// continue;
+// }
+
+ int num_blocks = data_length / block_size;
+ int number_of_insert_values = block_size - remainder;
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+
+ s = System.nanoTime();
+ int encode_pos = length;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encode_pos = length;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data2_arr[num_blocks * block_size + i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data2_arr, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+ }
+
+ e = System.nanoTime();
+ insertRemainderTime += ((e - s) / repeatTime);
+
+
+ // larger values
+
+ Random random = new Random();
+ int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+
+ int[] data_remainder = new int[remainder+1];
+ System.arraycopy(data2_arr, num_blocks * block_size, data_remainder, 0, remainder);
+ data_remainder[remainder] = randomNumber;
+
+ remainder += 1;
+
+
+ s = System.nanoTime();
+ encode_pos = length;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encode_pos = length;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data_remainder[i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data_remainder, 0, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertRemainderOneTime += ((e - s) / repeatTime);
+
+
+ long insertRemainderOneTimeSmaller = 0;
+
+ int randomNumberSmaller = random.nextInt(random_lower_bound);
+
+ data_remainder[remainder-1] = randomNumber;
+
+// remainder += 1;
+
+
+ s = System.nanoTime();
+ encode_pos = length;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encode_pos = length;
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data_remainder[i];
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data_remainder, 0, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertRemainderOneTimeSmaller += ((e - s) / repeatTime);
+
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertRemainderOneTime-insertRemainderTime),
+ String.valueOf(insertRemainderOneTimeSmaller-insertRemainderTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertSmallerTest.java
new file mode 100644
index 0000000..897efec
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateInsertSmallerTest.java
@@ -0,0 +1,1048 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class UpdateInsertSmallerTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+// updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_insert_smaller.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+// "Insert Time", // 拼接原始值的时间
+ "Insert Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long insertSmallerTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[3];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Query");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ if(remainder < 4){
+ continue;
+ }
+
+ int num_blocks = data_length / block_size;
+ int number_of_insert_values = block_size - remainder;
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+ Random random = new Random();
+ int randomNumber = random.nextInt(random_lower_bound);
+
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+// data2_arr[num_blocks * block_size + remainder] = randomNumber;
+//// for (int i = 0; i < number_of_insert_values; i++) {
+//// data2_arr[num_blocks * block_size + remainder + i] = remaining_values[i];
+//// }
+//// data2_arr[data1.size()] = updateRange.get(datasetName);
+//// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+// }
+
+// e = System.nanoTime();
+// appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int new_decode_posencode_pos = length;
+
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int encode_pos = length;
+
+ int beta_o = beta[0];
+
+ int new_block_encode_length = beta[2];
+ int m = bytes2Integer(encoded_result, new_block_encode_length+4, 1);
+ int bitwidth_random = bitWidth(randomNumber);
+ if(bitwidth_random <= m*beta_o){
+ // 计算要加几个sub-column
+// int number_of_sub_column_random = (bitwidth_random + beta_o - 1) / beta_o;
+// int add_number_of_sub_column = number_of_sub_column_random - m;
+ int mask = (1 << beta[0]) - 1;
+
+
+
+ int l = (m + beta_o - 1) / beta_o;
+
+ int[] bitWidthList = new int[l];
+
+ int new_decode_pos = decodeBitPacking(encoded_result, new_block_encode_length+6,
+ 8, l, bitWidthList);
+
+// int[] add_sub_columns = new int[number_of_sub_column_random];
+// int[] bit_width_add_sub_columns = new int[number_of_sub_column_random];
+ int[] add_sub_columns = new int[l];
+ int[] bit_width_add_sub_columns = new int[l];
+// boolean[] is_changed = new boolean[number_of_sub_column_random];
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta[0];
+ add_sub_columns[i] = (randomNumber >> shiftAmount) & mask;
+ int tmp_bit_width = bitWidth(add_sub_columns[i]);
+ bit_width_add_sub_columns[i] = Math.max(tmp_bit_width, bitWidthList[i]);
+// is_changed[i]= tmp_bit_width > bitWidthList[i];
+ }
+// for (int i=l; i<add_number_of_sub_column; i++){
+// bit_width_add_sub_columns[i] = bitWidth(add_sub_columns[i]);
+// }
+
+ int[][] subcolumnList = new int[l][remainder];
+ int[][] new_subcolumnList = new int[l][remainder+1];
+
+ int[] encodingType = new int[l];
+ int[] new_encodingType = new int[l];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, 1, l, encodingType);
+ System.arraycopy(encodingType, 0, new_encodingType, 0, l);
+// System.out.println(Arrays.toString(encodingType));
+
+ int bw = bitWidth(block_size);
+ byte[] new_encoded_result = new byte[(remainder+1)*16];
+ int pos_of_new_encoded_result = 0;
+
+ for(int j = l-1;j>=0;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ // if bit-packing
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, remainder,
+ subcolumnList[j]);
+
+// if(is_changed[j]){
+//
+//
+// }else{
+ System.arraycopy(subcolumnList[j], 0, new_subcolumnList[j], 0, remainder);
+ new_subcolumnList[j][remainder] = add_sub_columns[j];
+ pos_of_new_encoded_result = bitPacking(new_subcolumnList[j], bit_width_add_sub_columns[j],
+ pos_of_new_encoded_result, new_encoded_result, remainder+1);
+// }
+ }
+ else {
+// System.out.println(encoded_result[new_decode_pos]);
+// System.out.println(encoded_result[new_decode_pos+1]);
+ // if rle
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+
+ new_decode_pos += 2;
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bw, index, run_length);
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, index, rle_values);
+
+// System.out.println(Arrays.toString(run_length));
+// System.out.println(Arrays.toString(rle_values));
+ if(rle_values[index-1] == add_sub_columns[j]){
+ run_length[index-1] += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+ pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+
+ }else{
+ int[] new_run_length = new int[index+1];
+ int[] new_rle_values = new int[index+1];
+ System.arraycopy(new_run_length, 0, run_length, 0, index);
+ System.arraycopy(new_rle_values, 0, rle_values, 0, index);
+ new_run_length[index] = 1;
+ new_rle_values[index] = add_sub_columns[j];
+
+ index ++;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(new_run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+
+ pos_of_new_encoded_result = bitPacking(new_rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+
+
+ }
+
+ }
+ }
+// for (int j=l; j<add_number_of_sub_column; j++){
+// new_encodingType[j]=1;
+// int[] run_length = {remainder,1};
+// int[] rle_values = {0,add_sub_columns[j]};
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (2 >> 8);
+// pos_of_new_encoded_result += 1;
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (2 & 0xFF);
+// pos_of_new_encoded_result += 1;
+// pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, 2);
+// pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, 2);
+// }
+
+
+ }
+
+
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertSmallerTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertSmallerTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateLargerTest.java
new file mode 100644
index 0000000..9fbd7a8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateLargerTest.java
@@ -0,0 +1,1107 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class UpdateLargerTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+
+ public static int decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+ // ArrayList<Integer> result_list = new ArrayList<>();
+ // int[] result_list = new int[num_values];
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+ decode_pos += bit_width;
+ }
+
+ decode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+ decode_pos += bit_width;
+ }
+
+ return (decode_pos + 7) / 8;
+ }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+// System.out.println("------------------------------------------------");
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+// System.out.println(encode_pos);
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+// System.out.println(encode_pos);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+// System.out.println(encode_pos);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+ beta[3] = remainder;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ public static boolean compareBits(byte[] encoded_result, int new_decode_pos, int index, int bitWidth, int value) {
+ // 计算起始位位置
+ int bit_pos = bitWidth * index;
+
+ // 计算起始字节和位偏移
+ int startByte = new_decode_pos + (bit_pos / 8);
+ int bitOffset = bit_pos % 8;
+
+ // 确保值不会超出指定位数范围
+ int maskedValue = value & ((1 << bitWidth) - 1);
+
+ // 读取指定位段
+ int readValue = 0;
+ int bitsRemaining = bitWidth;
+ int currentByteIndex = startByte;
+ int currentBitOffset = bitOffset;
+
+ while (bitsRemaining > 0 && currentByteIndex < encoded_result.length) {
+ // 计算当前字节中可以读取的位数
+ int bitsInThisByte = Math.min(8 - currentBitOffset, bitsRemaining);
+
+ // 从当前字节提取指定位
+ int byteValue = encoded_result[currentByteIndex] & 0xFF;
+ int extractedBits = (byteValue >> (8 - currentBitOffset - bitsInThisByte)) & ((1 << bitsInThisByte) - 1);
+
+ // 将提取的位添加到结果中
+ readValue = (readValue << bitsInThisByte) | extractedBits;
+
+ // 更新计数器
+ bitsRemaining -= bitsInThisByte;
+ currentByteIndex++;
+ currentBitOffset = 0; // 后续字节从第0位开始
+ }
+
+ // 比较读取的值与给定值的低位
+ return readValue == maskedValue;
+ }
+ public static void updateBits(byte[] encoded_result, int new_decode_pos, int bit_pos, int bitWidth, int value) {
+ // 计算起始字节和位偏移
+ int startByte = new_decode_pos + (bit_pos / 8);
+ int bitOffset = bit_pos % 8;
+
+ // 确保值不会超出指定位数范围
+ int maskedValue = value & ((1 << bitWidth) - 1);
+
+ // 处理跨字节更新
+ int bitsRemaining = bitWidth;
+ int currentByteIndex = startByte;
+ int currentBitOffset = bitOffset;
+
+ while (bitsRemaining > 0) {
+ // 计算当前字节中可以更新的位数
+ int bitsInThisByte = Math.min(8 - currentBitOffset, bitsRemaining);
+
+ // 创建掩码：清除目标位
+ int clearMask = ~(((1 << bitsInThisByte) - 1) << (8 - currentBitOffset - bitsInThisByte));
+
+ // 准备要设置的值（移位到正确位置）
+ int valuePart = (maskedValue << (bitWidth - bitsRemaining)) >>> (bitWidth - bitsInThisByte);
+ int shiftedValue = valuePart << (8 - currentBitOffset - bitsInThisByte);
+
+ // 更新当前字节
+ encoded_result[currentByteIndex] = (byte) ((encoded_result[currentByteIndex] & clearMask) | shiftedValue);
+
+ // 更新计数器
+ bitsRemaining -= bitsInThisByte;
+ currentByteIndex++;
+ currentBitOffset = 0; // 后续字节从第0位开始
+ }
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+ updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 500;
+
+ repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_update_larger.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+// "Insert Time", // 拼接原始值的时间
+ "Insert Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+// if(!datasetName.equals("Stocks-USA")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long insertGreaterTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[4];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Update");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ if(remainder < 4){
+ continue;
+ }
+ Decoder(encoded_result);
+
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+ Random random = new Random();
+ int randomNumber =( random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound) << 2;
+
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+// data2_arr[num_blocks * block_size + remainder] = randomNumber;
+//// for (int i = 0; i < number_of_insert_values; i++) {
+//// data2_arr[num_blocks * block_size + remainder + i] = remaining_values[i];
+//// }
+//// data2_arr[data1.size()] = updateRange.get(datasetName);
+//// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+// }
+
+// e = System.nanoTime();
+// appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int new_decode_posencode_pos = length;
+
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int encode_pos = length;
+
+ int beta_o = beta[0];
+ int remaining_pos = beta[3];
+
+ //
+ int new_block_encode_length = beta[2];
+ int m = bytes2Integer(encoded_result, new_block_encode_length+4, 1);
+ int bitwidth_random = bitWidth(randomNumber);
+// System.out.println("bitwidth_random:"+bitwidth_random);
+// System.out.println("m:"+m);
+ if(bitwidth_random>m*beta_o){
+
+ System.out.println("updated");
+ // 计算要加几个sub-column
+ int number_of_sub_column_random = (bitwidth_random + beta_o - 1) / beta_o;
+ int add_number_of_sub_column = number_of_sub_column_random - m;
+ int mask = (1 << beta[0]) - 1;
+
+
+
+ int l = (m + beta_o - 1) / beta_o;
+
+ int[] bitWidthList = new int[l];
+
+ int new_decode_pos = decodeBitPacking(encoded_result, new_block_encode_length+6,
+ 8, l, bitWidthList);
+
+ int[] add_sub_columns = new int[number_of_sub_column_random];
+ int[] bit_width_add_sub_columns = new int[number_of_sub_column_random];
+// boolean[] is_changed = new boolean[number_of_sub_column_random];
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta[0];
+ add_sub_columns[i] = (randomNumber >> shiftAmount) & mask;
+ int tmp_bit_width = bitWidth(add_sub_columns[i]);
+ bit_width_add_sub_columns[i] = Math.max(tmp_bit_width, bitWidthList[i]);
+// is_changed[i]= tmp_bit_width > bitWidthList[i];
+ }
+ for (int i=l; i<add_number_of_sub_column; i++){
+ bit_width_add_sub_columns[i] = bitWidth(add_sub_columns[i]);
+ }
+
+// int[][] subcolumnList = new int[l][remainder];
+// int[][] new_subcolumnList = new int[l][remainder+1];
+
+ int[] encodingType = new int[l];
+ int[] new_encodingType = new int[number_of_sub_column_random];
+
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, 1, l, encodingType);
+ System.arraycopy(encodingType, 0, new_encodingType, 0, l);
+// System.out.println(Arrays.toString(encodingType));
+
+ int bw = bitWidth(block_size);
+ byte[] new_encoded_result = new byte[(remainder+1)*16];
+ int pos_of_new_encoded_result = 0;
+
+ for(int j = l-1;j>=0;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ // if bit-packing
+ int bit_pos = (remaining_pos-1)*bitWidth;
+ updateBits(encoded_result, new_decode_pos, bit_pos, bitWidth, add_sub_columns[j]);
+ new_decode_pos += ((bit_pos+bitWidth+7)/8);
+ }
+ else {
+// System.out.println(encoded_result[new_decode_pos]);
+// System.out.println(encoded_result[new_decode_pos+1]);
+ // if rle
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+
+ new_decode_pos += 2;
+
+ int pre_new_decode_pos = new_decode_pos;
+
+
+ new_decode_pos += (bw*index + 7)/8;
+ if(!compareBits(encoded_result, (new_decode_pos-1),index, bitWidth, add_sub_columns[j]) ){
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+ new_decode_pos = pre_new_decode_pos;
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bw, index, run_length);
+ new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, index, rle_values);
+ int[] new_run_length = new int[index+1];
+ int[] new_rle_values = new int[index+1];
+ System.arraycopy( run_length, 0, new_run_length, 0, index);
+ System.arraycopy( rle_values, 0, new_rle_values, 0, index);
+ new_run_length[index] = 1;
+ new_rle_values[index] = add_sub_columns[j];
+
+ index ++;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+ pos_of_new_encoded_result += 1;
+// System.out.println(index);
+// System.out.println(remainder);
+// System.out.println(bw);
+ pos_of_new_encoded_result = bitPacking(new_run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+ pos_of_new_encoded_result = bitPacking(new_rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+
+ }else{
+ new_decode_pos += (bitWidth*index + 7)/8;
+ }
+
+ }
+ }
+ for (int j=l; j<number_of_sub_column_random; j++){
+// System.out.println("add new subcolumn");
+ new_encodingType[j]=1;
+ int[] run_length = {remainder,1};
+ int[] rle_values = {0,add_sub_columns[j]};
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (2 >> 8);
+ pos_of_new_encoded_result += 1;
+ new_encoded_result[pos_of_new_encoded_result] = (byte) (2 & 0xFF);
+ pos_of_new_encoded_result += 1;
+ pos_of_new_encoded_result = bitPacking(run_length, bw, pos_of_new_encoded_result, new_encoded_result, 2);
+ pos_of_new_encoded_result = bitPacking(rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, 2);
+ }
+
+ }
+
+
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertGreaterTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertGreaterTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateSmallerTest.java
new file mode 100644
index 0000000..ff28139
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/UpdateSmallerTest.java
@@ -0,0 +1,1165 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+public class UpdateSmallerTest {
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static void intToBytes(int srcNum, byte[] result, int pos, int width) {
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ int mask = 1 << (8 - cnt);
+ cnt += m;
+ byte y = (byte) (srcNum >>> width);
+ y = (byte) (y << (8 - cnt));
+ mask = ~(mask - (1 << (8 - cnt)));
+ result[index] = (byte) (result[index] & mask | y);
+ srcNum = srcNum & ~(-1 << width);
+ if (cnt == 8) {
+ index++;
+ cnt = 0;
+ }
+ }
+ }
+
+ public static int bytesToInt(byte[] result, int pos, int width) {
+ int ret = 0;
+ int cnt = pos & 0x07;
+ int index = pos >> 3;
+ while (width > 0) {
+ int m = width + cnt >= 8 ? 8 - cnt : width;
+ width -= m;
+ ret = ret << m;
+ byte y = (byte) (result[index] & (0xff >> cnt));
+ y = (byte) ((y & 0xff) >>> (8 - cnt - m));
+ ret = ret | (y & 0xff);
+ cnt += m;
+ if (cnt == 8) {
+ cnt = 0;
+ index++;
+ }
+ }
+ return ret;
+ }
+
+ public static void boolToBytes(boolean value, byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ if (value) {
+ result[byteIndex] |= (1 << (7 - bitOffset));
+ } else {
+ result[byteIndex] &= ~(1 << (7 - bitOffset));
+ }
+ }
+
+ public static boolean bytesToBool(byte[] result, int pos) {
+ int byteIndex = pos >> 3;
+ int bitOffset = pos & 0x07;
+
+ return (result[byteIndex] & (1 << (7 - bitOffset))) != 0;
+ }
+
+ public static void pack8Values(int[] values, int offset, int width, int encode_pos,
+ byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset, int width, int[] result_list, int result_offset) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ // result_list.add((int) (buffer >>> (totalBits - width)));
+ result_list[result_offset + valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(int[] numbers, int bit_width, int encode_pos,
+ byte[] encoded_result, int num_values) {
+ int block_num = num_values / 8;
+ int remainder = num_values % 8;
+
+ for (int i = 0; i < block_num; i++) {
+ pack8Values(numbers, i * 8, bit_width, encode_pos, encoded_result);
+ encode_pos += bit_width;
+ }
+
+ encode_pos *= 8;
+
+ for (int i = 0; i < remainder; i++) {
+ intToBytes(numbers[block_num * 8 + i], encoded_result, encode_pos, bit_width);
+ encode_pos += bit_width;
+ }
+
+ return (encode_pos + 7) / 8;
+ }
+ public static int decodeBitPacking(byte[] encoded, int bytePos, int bitWidth, int numValues, int[] result) {
+ // 参数检查（可选，但推荐在 debug 时打开）
+// if (bitWidth <= 0 || bitWidth > 32) throw new IllegalArgumentException("bitWidth must be 1..32");
+ if (numValues == 0) return bytePos;
+
+ final long mask = (bitWidth == 32) ? 0xFFFFFFFFL : ((1L << bitWidth) - 1L);
+
+ int valuesWritten = 0;
+ int byteIndex = bytePos;
+ long bitBuffer = 0L; // buffer 存放尚未消费的 bits（放在低位或高位都可以，这里用“高位拼入、右移取值”方式）
+ int bitsInBuffer = 0; // buffer 中可用的位数
+
+ int fullBlocks = numValues >>> 3; // 每 block 8 个值
+ int rem = numValues & 7;
+
+ // 处理每个 block（每 block 有 8 个值）
+ for (int b = 0; b < fullBlocks; b++) {
+ for (int v = 0; v < 8; v++) {
+ // 保证 buffer 中至少有 bitWidth 位可供取值
+ while (bitsInBuffer < bitWidth) {
+ // 按大端拼入：左移 8 位再或入下一个字节
+ bitBuffer = (bitBuffer << 8) | (encoded[byteIndex++] & 0xFFL);
+ bitsInBuffer += 8;
+ }
+ int shift = bitsInBuffer - bitWidth; // 从高位取出这次要的 bitWidth 位
+ result[valuesWritten++] = (int) ((bitBuffer >>> shift) & mask);
+ // 删除已消费的高位 bits
+ bitsInBuffer -= bitWidth;
+ if (bitsInBuffer == 0) {
+ bitBuffer = 0L;
+ } else {
+ // 保留低 bitsInBuffer 位（把高位已经消费掉）
+ long keepMask = (bitsInBuffer == 64) ? ~0L : ((1L << bitsInBuffer) - 1L);
+ bitBuffer &= keepMask;
+ }
+ }
+ }
+
+ // 处理剩余值
+ for (int v = 0; v < rem; v++) {
+ while (bitsInBuffer < bitWidth) {
+ bitBuffer = (bitBuffer << 8) | (encoded[byteIndex++] & 0xFFL);
+ bitsInBuffer += 8;
+ }
+ int shift = bitsInBuffer - bitWidth;
+ result[valuesWritten++] = (int) ((bitBuffer >>> shift) & mask);
+ bitsInBuffer -= bitWidth;
+ if (bitsInBuffer == 0) {
+ bitBuffer = 0L;
+ } else {
+ long keepMask = (bitsInBuffer == 64) ? ~0L : ((1L << bitsInBuffer) - 1L);
+ bitBuffer &= keepMask;
+ }
+ }
+
+ // 返回已消费到的字节索引（下一个可读字节）
+ return byteIndex;
+ }
+
+// public static int decodeBitPacking(
+// byte[] encoded, int decode_pos, int bit_width, int num_values, int[] result_list) {
+// // ArrayList<Integer> result_list = new ArrayList<>();
+// // int[] result_list = new int[num_values];
+// int block_num = num_values>>3;
+// int remainder = num_values % 8;
+//
+// for (int i = 0; i < block_num; i++) { // bitpacking
+// unpack8Values(encoded, decode_pos, bit_width, result_list, i * 8);
+// decode_pos += bit_width;
+// }
+//
+// decode_pos *= 8;
+//
+// for (int i = 0; i < remainder; i++) {
+// result_list[block_num * 8 + i] = bytesToInt(encoded, decode_pos, bit_width);
+// decode_pos += bit_width;
+// }
+//
+// return (decode_pos + 7) >>3;
+// }
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ public static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int Subcolumn(int[] x, int x_length, int m, int block_size) {
+
+ int betaBest = 1;
+
+ int cMin = Integer.MAX_VALUE;
+
+ // int[] beta_list = {1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31};
+ // int[] beta_list = { 1, 2, 3, 5, 7, 11 };
+ // int[] beta_list = { 1, 2, 3, 4 };
+ int[] beta_list = { 2, 3, 4 };
+
+ int bw = bitWidth(block_size);
+
+ int[] bitWidthListList = new int[m];
+
+ for (int beta : beta_list) {
+ if (beta > m) {
+ break;
+ }
+ // System.out.println("beta: " + beta);
+
+ int l = (m + beta - 1) / beta;
+
+ // System.out.println("l: " + l);
+
+ int[][] subcolumnList = new int[l][x_length];
+
+ int cost = 0;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ for (int j = 0; j < x_length; j++) {
+ subcolumnList[i][j] = (x[j] >> (i * beta)) & ((1 << beta) - 1);
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthListList[i] = bitWidth(maxValuePart);
+ }
+
+ for (int i = 0; i < l; i++) {
+ int bpCost = bitWidthListList[i] * x_length;
+ int rleCost = 0;
+
+ // int count = 1;
+ int currentNumber = subcolumnList[i][0];
+
+ int index = 0;
+
+ boolean bpBest = false;
+
+ for (int j = 1; j < x_length; j++) {
+ if (subcolumnList[i][j] != currentNumber) {
+ index++;
+ currentNumber = subcolumnList[i][j];
+ }
+
+ if (bw * index + bitWidthListList[i] * index >= bpCost) {
+ bpBest = true;
+ break;
+ }
+ }
+
+ if (bpBest) {
+ cost += bpCost;
+ continue;
+ }
+
+ index++;
+
+ // System.out.println("index: " + index);
+
+ rleCost = bw * index + bitWidthListList[i] * index;
+
+ // System.out.println("bpCost: " + bpCost + " rleCost: " + rleCost);
+
+ if (bpCost <= rleCost) {
+ cost += bpCost;
+ } else {
+ cost += rleCost;
+ }
+ }
+
+ // System.out.println("cost: " + cost);
+
+ if (cost < cMin) {
+ cMin = cost;
+ betaBest = beta;
+ }
+ }
+
+ return betaBest;
+ }
+
+ public static int SubcolumnEncoder(int[] list, int encode_pos, byte[] encoded_result, int[] beta, int block_size) {
+ int list_length = list.length;
+ int maxValue = 0;
+ for (int i = 0; i < list_length; i++) {
+ if (list[i] > maxValue) {
+ maxValue = list[i];
+ }
+ }
+
+ int m = bitWidth(maxValue);
+
+ intByte2Bytes(m, encode_pos, encoded_result);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ // int[] bitWidthList = new int[m];
+
+ // int[][] subcolumnList = new int[m][list_length];
+
+ int l;
+
+ // int betaBest = beta[0];
+ // byte betaBest = (byte) beta[0];
+
+ l = (m + beta[0] - 1) / beta[0];
+
+ int[] bitWidthList = new int[l];
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ intByte2Bytes(beta[0], encode_pos, encoded_result);
+ encode_pos += 1;
+
+ int bw = bitWidth(block_size);
+ int mask = (1 << beta[0]) - 1;
+
+ for (int i = 0; i < l; i++) {
+ int maxValuePart = 0;
+ int shiftAmount = i * beta[0];
+ for (int j = 0; j < list_length; j++) {
+ subcolumnList[i][j] = (list[j] >> shiftAmount) & mask;
+ if (subcolumnList[i][j] > maxValuePart) {
+ maxValuePart = subcolumnList[i][j];
+ }
+ }
+ bitWidthList[i] = bitWidth(maxValuePart);
+ }
+
+ encode_pos = bitPacking(bitWidthList, 8, encode_pos, encoded_result, l);
+
+ int[] encodingType = new int[l];
+
+ // encoded_result 预留大小为 (l + 7) / 8 的大小，存储每个分列的类型
+ int preTypePos = encode_pos;
+ encode_pos += (l + 7) / 8;
+
+ for (int i = l - 1; i >= 0; i--) {
+ // 对于每个分列，计算使用 bit packing 还是 rle
+ int bpCost = bitWidthList[i] * list_length;
+ int rleCost = 0;
+
+ int previous = subcolumnList[i][0];
+ int index = 0;
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ index++;
+ previous = currentNumber;
+ }
+
+ if (bw * index + bitWidthList[i] * index >= bpCost) {
+ break;
+ }
+ }
+
+ index++;
+
+ rleCost = bw * index + bitWidthList[i] * index;
+
+ if (bpCost <= rleCost) {
+ encodingType[i] = 0;
+
+ encode_pos = bitPacking(subcolumnList[i], bitWidthList[i], encode_pos, encoded_result, list_length);
+
+ } else {
+ encodingType[i] = 1;
+
+ encoded_result[encode_pos] = (byte) (index >> 8);
+ encode_pos += 1;
+ encoded_result[encode_pos] = (byte) (index & 0xFF);
+ encode_pos += 1;
+
+ index = 0;
+ int[] run_length = new int[list_length];
+ int[] rle_values = new int[list_length];
+ previous = subcolumnList[i][0];
+
+ for (int j = 1; j < list_length; j++) {
+ int currentNumber = subcolumnList[i][j];
+ if (currentNumber != previous) {
+ run_length[index] = j;
+ rle_values[index] = previous;
+ index++;
+ previous = currentNumber;
+ }
+ }
+
+ run_length[index] = list_length;
+ rle_values[index] = previous;
+ index++;
+
+ encode_pos = bitPacking(run_length, bw, encode_pos, encoded_result, index);
+
+ encode_pos = bitPacking(rle_values, bitWidthList[i], encode_pos, encoded_result, index);
+
+ }
+
+ }
+
+ preTypePos = bitPacking(encodingType, 1, preTypePos, encoded_result, l);
+
+ return encode_pos;
+ }
+
+ public static int SubcolumnDecoder(byte[] encoded_result, int encode_pos, int[] list, int block_size) {
+ int list_length = list.length;
+
+ // int m = encoded_result[encode_pos];
+ int m = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ if (m == 0) {
+ return encode_pos;
+ }
+
+ int bw = bitWidth(block_size);
+
+ int beta = bytes2Integer(encoded_result, encode_pos, 1);
+ encode_pos += 1;
+
+ int l = (m + beta - 1) / beta;
+
+ int[] bitWidthList = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 8, l, bitWidthList);
+
+ int[][] subcolumnList = new int[l][list_length];
+
+ int[] encodingType = new int[l];
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, 1, l, encodingType);
+
+ for (int i = l - 1; i >= 0; i--) {
+ int type = encodingType[i];
+ int bitWidth = bitWidthList[i];
+ if (type == 0) {
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, list_length,
+ subcolumnList[i]);
+ } else {
+ int index = ((encoded_result[encode_pos] & 0xFF) << 8) | (encoded_result[encode_pos + 1] & 0xFF);
+
+ encode_pos += 2;
+
+// System.out.println("------------------------------------------------");
+
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+// System.out.println(encode_pos);
+
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bw, index, run_length);
+// System.out.println(encode_pos);
+ encode_pos = decodeBitPacking(encoded_result, encode_pos, bitWidth, index, rle_values);
+// System.out.println(encode_pos);
+
+ int currentIndex = 0;
+ for (int j = 0; j < index; j++) {
+ int endPos = run_length[j];
+ int value = rle_values[j];
+ while (currentIndex < endPos) {
+ subcolumnList[i][currentIndex] = value;
+ currentIndex++;
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < l; i++) {
+ int shiftAmount = i * beta;
+ for (int j = 0; j < list_length; j++) {
+ list[j] |= subcolumnList[i][j] << shiftAmount;
+ }
+ }
+
+ return encode_pos;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i * block_size;
+ int end = i * block_size + remaining;
+
+ for (int j = base; j < end; j++) {
+ int cur = ts_block[j];
+ if (cur < value_delta_min) {
+ value_delta_min = cur;
+ }
+ if (cur > value_delta_max) {
+ value_delta_max = cur;
+ }
+ }
+
+ for (int j = base; j < end; j++) {
+ ts_block_delta[j - base] = ts_block[j] - value_delta_min;
+ }
+
+ min_delta[0] = value_delta_min;
+
+ return ts_block_delta;
+ }
+
+ public static int BlockEncoder(int[] data, int block_index, int block_size, int remainder,
+ int encode_pos, byte[] encoded_result, int[] beta) {
+ int[] min_delta = new int[3];
+
+ int[] data_delta = getAbsDeltaTsBlock(data, block_index, block_size,
+ remainder, min_delta);
+
+ int2Bytes(min_delta[0], encode_pos, encoded_result);
+ encode_pos += 4;
+
+ if (block_index == 0) {
+ int maxValue = 0;
+ for (int j = 0; j < remainder; j++) {
+ if (data_delta[j] > maxValue) {
+ maxValue = data_delta[j];
+ }
+ }
+ int m = bitWidth(maxValue);
+
+ beta[0] = Subcolumn(data_delta, remainder, m, block_size);
+ }
+
+ encode_pos = SubcolumnEncoder(data_delta, encode_pos,
+ encoded_result, beta, block_size);
+
+ return encode_pos;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, int[] data) {
+ int[] min_delta = new int[3];
+
+ min_delta[0] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int[] block_data = new int[remainder];
+
+ encode_pos = SubcolumnDecoder(encoded_result, encode_pos,
+ block_data, block_size);
+
+ for (int i = 0; i < remainder; i++) {
+ data[block_index * block_size + i] = block_data[i] + min_delta[0];
+ }
+
+ return encode_pos;
+ }
+
+
+ public static int[] Decoder(byte[] encoded_result) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] data = new int[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ data[num_blocks * block_size + i] = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, data);
+ }
+
+ return data;
+ }
+
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ public static int Encoder(int[] data, int block_size, byte[] encoded_result, int[] beta) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+ int remainder = data_length % block_size;
+
+// int[] beta = new int[1];
+ beta[0] = 2;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result, beta);
+ }
+
+ int maximum = Integer.MIN_VALUE;
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+ if(maximum<value) maximum = value;
+ }
+ int max_bit_width = bitWidth(maximum);
+ int max_remainder = max_bit_width % beta[0];
+ int max_m = max_bit_width / beta[0];
+ if(max_remainder != 0){
+ max_m += 1;
+ }
+
+ int lower_bound = (int) (Math.pow(2,max_m * beta[0]));
+ beta[1] = lower_bound;
+ beta[2] = encode_pos;
+ beta[3] = remainder;
+
+ if (remainder <= 3) {
+ for (int i = 0; i < remainder; i++) {
+ int value = data[num_blocks * block_size + i];
+
+ int2Bytes(value, encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ } else {
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos,
+ encoded_result, beta);
+ }
+
+ // System.out.println("beta: " + beta[0]);
+
+ return encode_pos;
+ }
+ public static boolean compareBits(byte[] encoded_result, int new_decode_pos, int index, int bitWidth, int value) {
+ // 计算起始位位置
+ int bit_pos = bitWidth * index;
+
+ // 计算起始字节和位偏移
+ int startByte = new_decode_pos + (bit_pos / 8);
+ int bitOffset = bit_pos % 8;
+
+ // 确保值不会超出指定位数范围
+ int maskedValue = value & ((1 << bitWidth) - 1);
+
+ // 读取指定位段
+ int readValue = 0;
+ int bitsRemaining = bitWidth;
+ int currentByteIndex = startByte;
+ int currentBitOffset = bitOffset;
+
+ while (bitsRemaining > 0 && currentByteIndex < encoded_result.length) {
+ // 计算当前字节中可以读取的位数
+ int bitsInThisByte = Math.min(8 - currentBitOffset, bitsRemaining);
+
+ // 从当前字节提取指定位
+ int byteValue = encoded_result[currentByteIndex] & 0xFF;
+ int extractedBits = (byteValue >> (8 - currentBitOffset - bitsInThisByte)) & ((1 << bitsInThisByte) - 1);
+
+ // 将提取的位添加到结果中
+ readValue = (readValue << bitsInThisByte) | extractedBits;
+
+ // 更新计数器
+ bitsRemaining -= bitsInThisByte;
+ currentByteIndex++;
+ currentBitOffset = 0; // 后续字节从第0位开始
+ }
+
+ // 比较读取的值与给定值的低位
+ return readValue == maskedValue;
+ }
+ public static void updateBits(byte[] encoded_result, int new_decode_pos, int bit_pos, int bitWidth, int value) {
+ // 计算起始字节和位偏移
+ int startByte = new_decode_pos + (bit_pos / 8);
+ int bitOffset = bit_pos % 8;
+
+ // 确保值不会超出指定位数范围
+ int maskedValue = value & ((1 << bitWidth) - 1);
+
+ // 处理跨字节更新
+ int bitsRemaining = bitWidth;
+ int currentByteIndex = startByte;
+ int currentBitOffset = bitOffset;
+
+ while (bitsRemaining > 0) {
+ // 计算当前字节中可以更新的位数
+ int bitsInThisByte = Math.min(8 - currentBitOffset, bitsRemaining);
+
+ // 创建掩码：清除目标位
+ int clearMask = ~(((1 << bitsInThisByte) - 1) << (8 - currentBitOffset - bitsInThisByte));
+
+ // 准备要设置的值（移位到正确位置）
+ int valuePart = (maskedValue << (bitWidth - bitsRemaining)) >>> (bitWidth - bitsInThisByte);
+ int shiftedValue = valuePart << (8 - currentBitOffset - bitsInThisByte);
+
+ // 更新当前字节
+ encoded_result[currentByteIndex] = (byte) ((encoded_result[currentByteIndex] & clearMask) | shiftedValue);
+
+ // 更新计数器
+ bitsRemaining -= bitsInThisByte;
+ currentByteIndex++;
+ currentBitOffset = 0; // 后续字节从第0位开始
+ }
+ }
+ @Test
+ public void testQuery() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/"; //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+ String output_parent_dir = parent_dir + "result/update/";
+ // String output_parent_dir = parent_dir + "result/query_vs_beta/";
+
+ int block_size = 512;
+
+ HashMap<String, Integer> updateRange = new HashMap<>();
+
+ updateRange.put("Bird-migration", 2500000);
+ updateRange.put("Bitcoin-price", 160000000);
+ updateRange.put("City-temp", 480);
+ updateRange.put("Dewpoint-temp", 9500);
+ updateRange.put("IR-bio-temp", -300);
+ updateRange.put("PM10-dust", 1000);
+ updateRange.put("Stocks-DE", 40000);
+ updateRange.put("Stocks-UK", 20000);
+ updateRange.put("Stocks-USA", 5000);
+ updateRange.put("Wind-Speed", 50);
+ updateRange.put("Wine-Tasting", 0);
+
+ int repeatTime = 500;
+
+ // repeatTime = 1;
+
+ List<String> integerDatasets = new ArrayList<>();
+ integerDatasets.add("Wine-Tasting");
+
+// int beta = 1;
+ String outputPath = output_parent_dir + "subcolumn_update_smaller.csv";
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+// "Insert Time", // 拼接原始值的时间
+ "Insert Time with Sub-column", //拼接压缩后的数据的时间
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if(datasetName.equals("POI-lon") || datasetName.equals("POI-lat"))
+ continue;
+// if(!datasetName.equals("Stocks-USA")) continue;
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal)
+ max_decimal = cur_decimal;
+ data1.add(Float.valueOf(f_str));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()+1];
+ int max_mul = (int) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (int) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 4];
+
+ long encodeTime = 0;
+ long appendTime = 0;
+ long insertGreaterTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+ int[] beta = new int[4];
+ beta[0] = 2;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ length = Encoder(data2_arr, block_size, encoded_result,beta);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ double ratioTmp;
+
+ if (integerDatasets.contains(datasetName)) {
+ ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ } else {
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+ }
+
+ System.out.println("Update");
+
+ int data_length = data2_arr.length;
+ int remainder = data_length % block_size;
+ if(remainder < 4){
+ continue;
+ }
+ Decoder(encoded_result);
+
+// s = System.nanoTime();
+
+ int random_lower_bound = beta[1];
+// int[] remaining_values = new int[number_of_insert_values];
+
+// for (int i = 0; i < number_of_insert_values; i++) {
+// Random random = new Random();
+// int randomNumber = random.nextInt(Integer.MAX_VALUE - random_lower_bound) + random_lower_bound;
+// remaining_values[i] = randomNumber;
+// }
+ Random random = new Random();
+// Random random = new Random();
+ int randomNumber = random.nextInt(Math.min(Math.max(random_lower_bound/4,16),random_lower_bound));
+
+// for (int repeat = 0; repeat < repeatTime; repeat++) {
+// data2_arr[num_blocks * block_size + remainder] = randomNumber;
+//// for (int i = 0; i < number_of_insert_values; i++) {
+//// data2_arr[num_blocks * block_size + remainder + i] = remaining_values[i];
+//// }
+//// data2_arr[data1.size()] = updateRange.get(datasetName);
+//// UpdateAppendTest.Query(encoded_result, updateRange.get(datasetName));
+// }
+
+// e = System.nanoTime();
+// appendTime += ((e - s) / repeatTime);
+
+ s = System.nanoTime();
+
+ int new_decode_posencode_pos = length;
+
+ int beta_o = beta[0];
+ int remaining_pos = beta[3];
+
+ //
+ int new_block_encode_length = beta[2];
+ int m = bytes2Integer(encoded_result, new_block_encode_length+4, 1);
+ int bitwidth_random = bitWidth(randomNumber);
+ int mask = (1 << beta_o) - 1;
+ int l = (m + beta_o - 1) / beta_o;
+ int number_of_sub_column_random = (bitwidth_random + beta_o - 1) / beta_o;
+ int[] bitWidthList = new int[l];
+ int bw = bitWidth(block_size);
+ int new_decode_pos = decodeBitPacking(encoded_result, new_block_encode_length+6,
+ 8, l, bitWidthList);
+
+ int[] add_sub_columns = new int[l];
+ for (int i = 0; i < number_of_sub_column_random; i++) {
+ int shiftAmount = i * beta_o;
+ add_sub_columns[i] = (randomNumber >> shiftAmount) & mask;
+// int tmp_bit_width = bitWidth(add_sub_columns[i]);
+// bit_width_add_sub_columns[i] = Math.max(tmp_bit_width, bitWidthList[i]);
+// is_changed[i]= tmp_bit_width > bitWidthList[i];
+ }
+ long start_part3 = System.nanoTime();
+ int[] encodingType = new int[l];
+// int[] new_encodingType = new int[l];
+
+ int start_new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, 1, l, encodingType);
+// System.arraycopy(encodingType, 0, new_encodingType, 0, l);
+// System.out.println(Arrays.toString(encodingType));
+
+ long end_part3 = System.nanoTime();
+// part3_time += (end_part3-start_part3);
+ long part1_time = 0;
+ long part2_time = 0;
+// long part3_time = 0;
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+// int encode_pos = length;
+
+
+// int add_number_of_sub_column = number_of_sub_column_random - m;
+
+
+ new_decode_pos = start_new_decode_pos;
+
+
+ byte[] new_encoded_result = new byte[(remainder+1)*16];
+ int pos_of_new_encoded_result = 0;
+
+
+ for(int j=l-1;j>=number_of_sub_column_random;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ new_decode_pos += ((remaining_pos*bitWidth+7)/8);
+ }
+ else {
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+ new_decode_pos += 2;
+ new_decode_pos += ((bw*index + 7) >>3);
+ new_decode_pos += ((bitWidth*index + 7)>>3);
+ }
+ }
+
+ for(int j = number_of_sub_column_random-1;j>=0;j--){
+ int bitWidth = bitWidthList[j];
+ if(encodingType[j] == 0){
+ long start = System.nanoTime();
+ // if bit-packing
+ int bit_pos = (remaining_pos-1)*bitWidth;
+ updateBits(encoded_result, new_decode_pos, bit_pos, bitWidth, add_sub_columns[j]);
+ new_decode_pos += ((bit_pos+bitWidth+7) >> 3);
+ long end = System.nanoTime();
+ part1_time += (end-start);
+ }
+ else {
+// System.out.println(encoded_result[new_decode_pos]);
+// System.out.println(encoded_result[new_decode_pos+1]);
+ // if rle
+ long start = System.nanoTime();
+ int index = ((encoded_result[new_decode_pos] & 0xFF) << 8) | (encoded_result[new_decode_pos + 1] & 0xFF);
+
+ new_decode_pos += 2;
+
+ int pre_new_decode_pos = new_decode_pos;
+
+
+ new_decode_pos += (bw*index + 7)/8;
+ if(!compareBits(encoded_result, (new_decode_pos-1),index, bitWidth, add_sub_columns[j]) ){
+ int[] run_length = new int[index];
+ int[] rle_values = new int[index];
+// new_decode_pos = pre_new_decode_pos;
+// new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bw, index, run_length);
+// new_decode_pos = decodeBitPacking(encoded_result, new_decode_pos, bitWidth, index, rle_values);
+// int[] new_run_length = new int[index+1];
+// int[] new_rle_values = new int[index+1];
+// System.arraycopy( run_length, 0, new_run_length, 0, index);
+// System.arraycopy( rle_values, 0, new_rle_values, 0, index);
+// new_run_length[index] = 1;
+// new_rle_values[index] = add_sub_columns[j];
+//
+// index ++;
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (index >> 8);
+// pos_of_new_encoded_result += 1;
+// new_encoded_result[pos_of_new_encoded_result] = (byte) (index & 0xFF);
+// pos_of_new_encoded_result += 1;
+
+// pos_of_new_encoded_result = bitPacking(new_run_length, bw, pos_of_new_encoded_result, new_encoded_result, index);
+// pos_of_new_encoded_result = bitPacking(new_rle_values, bit_width_add_sub_columns[j], pos_of_new_encoded_result, new_encoded_result, index);
+ new_decode_pos += ((bitWidth*index + 7)>>3);
+ }else{
+ new_decode_pos += ((bitWidth*index + 7)>>3);
+ }
+ long end = System.nanoTime();
+ part2_time += (end-start);
+
+ }
+ }
+
+
+ }
+// length = encode_pos ;
+
+ e = System.nanoTime();
+ insertGreaterTime += ((e - s) / repeatTime);
+ System.out.println("part1: "+ part1_time/repeatTime);
+ System.out.println("part2: "+ part2_time/repeatTime);
+// System.out.println("part3: "+ part3_time/repeatTime);
+ System.out.println("insertSmallerTime: "+ insertGreaterTime);
+
+ String[] record = {
+ datasetName,
+ "Sub-columns",
+ String.valueOf(encodeTime),
+// String.valueOf(appendTime),
+ String.valueOf(insertGreaterTime),
+ String.valueOf(data1.size()),
+ String.valueOf(remainder),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+
+// System.out.println("beta: " + beta);
+
+ System.out.println(ratio);
+ }
+
+ writer.close();
+
+ }
+
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLong.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLong.java
new file mode 100644
index 0000000..341a2e3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLong.java
@@ -0,0 +1,348 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+public class VBPIndexLong {
+
+ public static final int W = 64;
+
+ public final int k;
+ public final int n;
+ public final int wordsPerPlane;
+ public final long[][] planes;
+
+ public enum Op {
+ EQ, NE, LT, LE, GT, GE
+ }
+
+ public VBPIndexLong(int kBits, long[] codes) {
+
+ this.k = kBits;
+ this.n = codes.length;
+ this.wordsPerPlane = (n + W - 1) / W;
+ this.planes = new long[k][wordsPerPlane];
+ pack(codes);
+ }
+
+ private void pack(long[] codes) {
+ for (int row = 0; row < n; row++) {
+ int wordIdx = row / W;
+ int bitPos = row % W;
+ long bitMask = 1L << bitPos;
+ long code = codes[row];
+ for (int t = 0; t < k; t++) {
+ if (((code >>> t) & 1) != 0) {
+ planes[t][wordIdx] |= bitMask;
+ }
+ }
+ }
+ }
+
+ public int[] select(Op op, long C) {
+ long codeMask = (k == 64) ? ~0L : ((1L << k) - 1L);
+ long Ck = (C & codeMask);
+ // return selectInternal(op, Ck);
+
+ // return selectInternal2(op, Ck);
+ return selectInternal3(op, Ck);
+ }
+
+ private int[] selectInternal2(Op op, long Ck) {
+ List<Integer> out = new ArrayList<>();
+ for (int row = 0; row < n; row++) {
+ long code = 0;
+ for (int t = 0; t < k; t++) {
+ int wordIdx = row / W;
+ int bitPos = row % W;
+ long bitVal = (planes[t][wordIdx] >> bitPos) & 1L;
+ code |= (bitVal << t);
+ }
+ switch (op) {
+ case EQ:
+ if (code == Ck)
+ out.add(row);
+ break;
+ case NE:
+ if (code != Ck)
+ out.add(row);
+ break;
+ case LT:
+ if (code < Ck)
+ out.add(row);
+ break;
+ case LE:
+ if (code <= Ck)
+ out.add(row);
+ break;
+ case GT:
+ if (code > Ck)
+ out.add(row);
+ break;
+ case GE:
+ if (code >= Ck)
+ out.add(row);
+ break;
+ }
+ }
+ return out.stream().mapToInt(i -> i).toArray();
+ }
+
+ private int[] selectInternal3(Op op, long Ck) {
+ List<Integer> out = new ArrayList<>();
+ for (int row = 0; row < n; row++) {
+ int wordIdx = row / W;
+ int bitPos = row % W;
+ switch (op) {
+ case EQ:
+ boolean match = true;
+ for (int t = 0; t < k; t++) {
+ long bitVal = (planes[t][wordIdx] >> bitPos) & 1L;
+ long cBit = (Ck >> t) & 1L;
+ if (bitVal != cBit) {
+ match = false;
+ break;
+ }
+ }
+ if (match) {
+ out.add(row);
+ }
+ break;
+ case NE:
+ boolean notMatch = false;
+ for (int t = 0; t < k; t++) {
+ long bitVal = (planes[t][wordIdx] >> bitPos) & 1L;
+ long cBit = (Ck >> t) & 1L;
+ if (bitVal != cBit) {
+ notMatch = true;
+ break;
+ }
+ }
+ if (notMatch) {
+ out.add(row);
+ }
+ break;
+ default:
+ long code = 0;
+ for (int t = 0; t < k; t++) {
+ long bitVal = (planes[t][wordIdx] >> bitPos) & 1L;
+ code |= (bitVal << t);
+ }
+ switch (op) {
+ case LT:
+ if (code < Ck)
+ out.add(row);
+ break;
+ case LE:
+ if (code <= Ck)
+ out.add(row);
+ break;
+ case GT:
+ if (code > Ck)
+ out.add(row);
+ break;
+ case GE:
+ if (code >= Ck)
+ out.add(row);
+ break;
+ }
+ }
+ }
+ return out.stream().mapToInt(i -> i).toArray();
+ }
+
+ private int[] selectInternal(Op op, long Ck) {
+ List<Integer> out = new ArrayList<>();
+
+ for (int w = 0; w < wordsPerPlane; w++) {
+ int bitsInThisWord = Math.min(W, n - w * W);
+ long validMask = (bitsInThisWord == 64) ? ~0L : ((1L << bitsInThisWord) - 1L);
+
+ long E = validMask;
+ long L = 0L;
+ long G = 0L;
+
+ for (int t = k - 1; t >= 0; t--) {
+ long B = planes[t][w] & validMask;
+ long cb = (Ck >>> t) & 1;
+ if (cb == 1) {
+ L |= (E & (~B));
+ E &= B;
+ } else {
+ G |= (E & B);
+ E &= (~B);
+ }
+ }
+
+ long res;
+ switch (op) {
+ case EQ:
+ res = E;
+ break;
+ case NE:
+ res = (~E) & validMask;
+ break;
+ case LT:
+ res = L;
+ break;
+ case LE:
+ res = (L | E) & validMask;
+ break;
+ case GT:
+ res = G;
+ break;
+ case GE:
+ res = (G | E) & validMask;
+ break;
+ default:
+ res = 0L;
+ }
+
+ int base = w * W;
+ long tmp = res;
+ while (tmp != 0L) {
+ int t = Long.numberOfTrailingZeros(tmp);
+ out.add(base + t);
+ tmp &= (tmp - 1);
+ }
+ }
+
+ return out.stream().mapToInt(i -> i).toArray();
+ }
+
+ public int count(Op op, long C) {
+ int[] res = select(op, C);
+ return res.length;
+ }
+
+ public int count() {
+ return n;
+ }
+
+ public int size() {
+ return n;
+ }
+
+ public long getCode(int row) {
+ if (row < 0 || row >= n)
+ throw new IndexOutOfBoundsException();
+ int wordIdx = row / W;
+ int bitPos = row % W;
+ long code = 0;
+ long mask = 1L << bitPos;
+ for (int t = 0; t < k; t++) {
+ long planeWord = planes[t][wordIdx];
+ if ((planeWord & mask) != 0) {
+ code |= (1L << t);
+ }
+ }
+ return code;
+ }
+
+ public int findMaxIndex() {
+ if (n == 0)
+ return -1;
+
+ boolean[] candidates = new boolean[n];
+ for (int i = 0; i < n; i++) {
+ candidates[i] = true;
+ }
+
+ for (int t = k - 1; t >= 0; t--) {
+ boolean hasOnes = false;
+
+ for (int row = 0; row < n; row++) {
+ if (candidates[row]) {
+ if ((planes[t][row / W] & (1L << (row % W))) != 0) {
+ hasOnes = true;
+ } else {
+ candidates[row] = false;
+ }
+ }
+ }
+
+ if (!hasOnes) {
+ for (int row = 0; row < n; row++) {
+ if (candidates[row]) {
+ if ((planes[t][row / W] & (1L << (row % W))) != 0) {
+ candidates[row] = false;
+ }
+ }
+ }
+ }
+ }
+
+ for (int row = 0; row < n; row++) {
+ if (candidates[row]) {
+ return row;
+ }
+ }
+
+ return -1;
+ }
+
+ public long sum() {
+ if (n == 0)
+ return 0L;
+
+ long totalSum = 0L;
+
+ for (int t = 0; t < k; t++) {
+ long bitContribution = 0L;
+
+ for (int i = 0; i < n; i++) {
+ long planeWord = planes[t][i / W];
+ bitContribution += (planeWord >> (i % W)) & 1;
+ }
+
+ totalSum += (bitContribution << t);
+ }
+
+ return totalSum;
+ }
+
+ public static void main(String[] args) {
+ int k = 3;
+ long[] codes = { 1, 5, 6, 1, 6, 4, 0, 7, 4, 3 };
+ VBPIndexLong idx = new VBPIndexLong(k, codes);
+
+ for (int i = 0; i < idx.wordsPerPlane; i++) {
+ System.out.println("word " + i + ":");
+ for (int t = 0; t < k; t++) {
+ System.out.println(String.format("%64s", Long.toBinaryString(idx.planes[t][i])).replace(' ', '0'));
+ }
+ System.out.println();
+ }
+
+ System.out.println("n = " + idx.size());
+
+ int[] lt4 = idx.select(Op.LT, 4);
+ System.out.print("< 4 -> ");
+ for (int i = 0; i < lt4.length; i++) {
+ System.out.print(lt4[i] + " ");
+ }
+ System.out.println();
+
+ int[] eq4 = idx.select(Op.EQ, 4);
+ System.out.print("= 4 -> ");
+ for (int i = 0; i < eq4.length; i++) {
+ System.out.print(eq4[i] + " ");
+ }
+ System.out.println();
+
+ int[] ge6 = idx.select(Op.GE, 6);
+ System.out.print(">= 6 -> ");
+ for (int i = 0; i < ge6.length; i++) {
+ System.out.print(ge6[i] + " ");
+ }
+ System.out.println();
+
+ for (int i = 0; i < idx.size(); i++) {
+ System.out.print(idx.getCode(i) + " ");
+ }
+ System.out.println();
+
+ System.out.println("count(<5) = " + idx.count(Op.LT, 5));
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLongTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLongTest.java
new file mode 100644
index 0000000..5519270
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPIndexLongTest.java
@@ -0,0 +1,325 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+import org.junit.Test;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+
+public class VBPIndexLongTest {
+
+ public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 3] = (byte) (integer);
+ }
+
+ public static void long2Bytes(long integer, int encode_pos, byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 56);
+ cur_byte[encode_pos + 1] = (byte) (integer >> 48);
+ cur_byte[encode_pos + 2] = (byte) (integer >> 40);
+ cur_byte[encode_pos + 3] = (byte) (integer >> 32);
+ cur_byte[encode_pos + 4] = (byte) (integer >> 24);
+ cur_byte[encode_pos + 5] = (byte) (integer >> 16);
+ cur_byte[encode_pos + 6] = (byte) (integer >> 8);
+ cur_byte[encode_pos + 7] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int bitWidth(int value) {
+ return 32 - Integer.numberOfLeadingZeros(value);
+ }
+
+ public static int bitWidth(long value) {
+ return 64 - Long.numberOfLeadingZeros(value);
+ }
+
+ public static int BlockEncoder(long[] data, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, byte[] encoded_result) {
+
+ long[] block_data = new long[remainder];
+ System.arraycopy(data, block_index * block_size, block_data, 0, remainder);
+
+ long min_value = Long.MAX_VALUE;
+ long max_value = Long.MIN_VALUE;
+ for (long value : block_data) {
+ if (value < min_value) {
+ min_value = value;
+ }
+ if (value > max_value) {
+ max_value = value;
+ }
+ }
+
+ for (int i = 0; i < remainder; i++) {
+ block_data[i] -= min_value;
+ }
+
+ long2Bytes(min_value, encode_pos, encoded_result);
+ encode_pos += 8;
+
+ int bw = bitWidth(max_value - min_value);
+
+ int2Bytes(bw, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ VBPIndexLong idx = new VBPIndexLong(bw, block_data);
+ indexList.add(idx);
+
+ return encode_pos;
+
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, long[] data) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ for (int i = 0; i < remainder; i++) {
+ long value = idx.getCode(i);
+
+ data[block_index * block_size + i] = value + min_value;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static int Encoder(long[] data, int block_size, ArrayList<VBPIndexLong> indexList, byte[] encoded_result) {
+ int data_length = data.length;
+ int encode_pos = 0;
+
+ int2Bytes(data_length, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int2Bytes(block_size, encode_pos, encoded_result);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int remainder = data_length % block_size;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockEncoder(data, i, block_size, block_size, encode_pos, indexList, encoded_result);
+ }
+
+ encode_pos = BlockEncoder(data, num_blocks, block_size, remainder, encode_pos, indexList,
+ encoded_result);
+
+ return encode_pos;
+ }
+
+ public static long[] Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] data = new long[data_length];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, data);
+ }
+
+ int remainder = data_length % block_size;
+
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, data);
+
+ return data;
+ }
+
+ public static int getDecimalPrecision(String str) {
+ int decimalIndex = str.indexOf(".");
+
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "path/to/your/directory/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = parent_dir + "result/";
+
+ String outputPath = output_parent_dir + "vbp.csv";
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<VBPIndexLong> indexList = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ indexList.clear();
+
+ length = Encoder(data2_arr, block_size, indexList, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (VBPIndexLong idx : indexList) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ data2_arr_decoded = Decoder(encoded_result, indexList);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPMaterializeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPMaterializeTest.java
new file mode 100644
index 0000000..ca25c72
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPMaterializeTest.java
@@ -0,0 +1,104 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPMaterializeTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // BitSet bitset_result = idx.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ // for (int i = 0; i < bitset_result.length(); i++) {
+ // if (bitset_result.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result = idx.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ for (int i = 0; i < query_result.length; i++) {
+ result[result_length[0]] = query_result[i] + (block_index * block_size);
+ result_length[0]++;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range, int[] result, int[] result_length) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ result_length[0] = 0;
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCount2Test.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCount2Test.java
new file mode 100644
index 0000000..bd911d7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCount2Test.java
@@ -0,0 +1,91 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryCount2Test {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length) {
+
+ encode_pos += 8;
+
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ int count = idx.count();
+
+ result[result_length[0]] = count;
+ result_length[0]++;
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCountTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCountTest.java
new file mode 100644
index 0000000..9f0883c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryCountTest.java
@@ -0,0 +1,95 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryCountTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+
+ int count = idx.count(VBPIndexLong.Op.EQ, bound_query_range);
+
+ result[result_length[0]] = count;
+ result_length[0]++;
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryEqualTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryEqualTest.java
new file mode 100644
index 0000000..ba3604b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryEqualTest.java
@@ -0,0 +1,105 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryEqualTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // BitSet bitset_result = idx.select(VBPIndexLong.Op.EQ, bound_query_range);
+
+ // for (int i = 0; i < bitset_result.length(); i++) {
+ // if (bitset_result.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result = idx.select(VBPIndexLong.Op.EQ, bound_query_range);
+
+ for (int i = 0; i < query_result.length; i++) {
+ result[result_length[0]] = query_result[i] + (block_index * block_size);
+ result_length[0]++;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterLessTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterLessTest.java
new file mode 100644
index 0000000..e76f3c8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterLessTest.java
@@ -0,0 +1,120 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryGreaterLessTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range, int bound_query_less_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // BitSet bitset_result = idx.select(VBPIndexLong.Op.GT, bound_query_range);
+
+ // BitSet bitset_result_less = idx.select(VBPIndexLong.Op.LT,
+ // bound_query_less_range);
+
+ // for (int i = 0; i < bitset_result.length(); i++) {
+ // if (bitset_result.get(i) && bitset_result_less.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result = idx.select(VBPIndexLong.Op.GT, bound_query_range);
+ int[] query_result_less = idx.select(VBPIndexLong.Op.LT, bound_query_less_range);
+
+ int i = 0, j = 0;
+
+ while (i < query_result.length && j < query_result_less.length) {
+ if (query_result[i] == query_result_less[j]) {
+ result[result_length[0]] = query_result[i] + (block_index * block_size);
+ result_length[0]++;
+ i++;
+ j++;
+ } else if (query_result[i] < query_result_less[j]) {
+ i++;
+ } else {
+ j++;
+ }
+ }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range,
+ int bound_query_less_range) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range, bound_query_less_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range, bound_query_less_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterTest.java
new file mode 100644
index 0000000..ff63c86
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGreaterTest.java
@@ -0,0 +1,105 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryGreaterTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // BitSet bitset_result = idx.select(VBPIndexLong.Op.GT, bound_query_range);
+
+ // for (int i = 0; i < bitset_result.length(); i++) {
+ // if (bitset_result.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result = idx.select(VBPIndexLong.Op.GT, bound_query_range);
+
+ for (int i = 0; i < query_result.length; i++) {
+ result[result_length[0]] = query_result[i] + (block_index * block_size);
+ result_length[0]++;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGroupTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGroupTest.java
new file mode 100644
index 0000000..a7c161f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryGroupTest.java
@@ -0,0 +1,196 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPQueryGroupTest {
+
+ private static final int[] BLOCK_SIZES = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+ private static final int TARGET_GROUP_COUNT = 20;
+
+ private static class RangeGroupConfig {
+ long start;
+ long width;
+ int groupCount;
+ }
+
+ public static int[] queryGroupCountByValueRangeByDecode(
+ byte[] encodedResult,
+ ArrayList<VBPIndexLong> indexList,
+ long rangeStart,
+ long rangeWidth,
+ int bucketCount) {
+ long[] decoded = VBPIndexLongTest.Decoder(encodedResult, indexList);
+ int[] result = new int[bucketCount];
+ for (long value : decoded) {
+ long bucket = Math.floorDiv(value - rangeStart, rangeWidth);
+ if (bucket >= 0 && bucket < bucketCount) {
+ result[(int) bucket]++;
+ }
+ }
+ return result;
+ }
+
+ private static RangeGroupConfig buildRangeGroupConfig(long[] dataArr) {
+ long min = Long.MAX_VALUE;
+ long max = Long.MIN_VALUE;
+ for (long value : dataArr) {
+ if (value < min) {
+ min = value;
+ }
+ if (value > max) {
+ max = value;
+ }
+ }
+
+ long span = max - min + 1L;
+ long width = Math.max(1L, (span + TARGET_GROUP_COUNT - 1L) / TARGET_GROUP_COUNT);
+ // long nice = niceWidth(width);
+ // Keep bucket count as close as possible to TARGET_GROUP_COUNT.
+ int groupCount = TARGET_GROUP_COUNT;
+
+ RangeGroupConfig config = new RangeGroupConfig();
+ config.start = min;
+ config.width = width;
+ config.groupCount = groupCount;
+ return config;
+ }
+
+ /*
+ private static long niceWidth(long rawWidth) {
+ long scale = 1L;
+ while (rawWidth >= 10L) {
+ rawWidth = (rawWidth + 9L) / 10L;
+ scale *= 10L;
+ }
+ if (rawWidth <= 1L) {
+ return scale;
+ }
+ if (rawWidth <= 2L) {
+ return 2L * scale;
+ }
+ if (rawWidth <= 5L) {
+ return 5L * scale;
+ }
+ return 10L * scale;
+ }
+ */
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "vbp_query_group_range_count.csv";
+
+ int repeatTime = 100;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Block Size",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] dataArr = new long[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (long) (data.get(i) * maxMul);
+ }
+ RangeGroupConfig groupConfig = buildRangeGroupConfig(dataArr);
+ System.out.println(
+ "group range config: start="
+ + groupConfig.start
+ + ", width="
+ + groupConfig.width
+ + ", groupCount="
+ + groupConfig.groupCount);
+
+ for (int blockSize : BLOCK_SIZES) {
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 12)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ ArrayList<VBPIndexLong> tmpIndexList = new ArrayList<>();
+ length = VBPIndexLongTest.Encoder(dataArr, blockSize, tmpIndexList, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ ArrayList<VBPIndexLong> indexList = new ArrayList<>();
+ length = VBPIndexLongTest.Encoder(dataArr, blockSize, indexList, encodedResult);
+
+ int[] groupResult = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ groupResult =
+ queryGroupCountByValueRangeByDecode(
+ encodedResult,
+ indexList,
+ groupConfig.start,
+ groupConfig.width,
+ groupConfig.groupCount);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println(
+ "blockSize=" + blockSize + ", groupCount: " + (groupResult == null ? 0 : groupResult.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(blockSize),
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMain.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMain.java
new file mode 100644
index 0000000..88dca9a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMain.java
@@ -0,0 +1,629 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryMain {
+
+ public static int getDecimalPrecision(String str) {
+ // 查找小数点的位置
+ int decimalIndex = str.indexOf(".");
+
+ // 如果没有小数点，精度为0
+ if (decimalIndex == -1) {
+ return 0;
+ }
+
+ // 获取小数点后的部分并返回其长度
+ return str.substring(decimalIndex + 1).length();
+ }
+
+ public static String extractFileName(String path) {
+ if (path == null || path.isEmpty()) {
+ return "";
+ }
+
+ File file = new File(path);
+ String fileName = file.getName();
+
+ int dotIndex = fileName.lastIndexOf('.');
+
+ if (dotIndex == -1 || dotIndex == 0) {
+ return fileName;
+ }
+
+ return fileName.substring(0, dotIndex);
+ }
+
+ @Test
+ public void test0() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ String input_parent_dir = parent_dir + "dataset/";
+
+ String output_parent_dir = "D:/encoding-subcolumn/result/vbp_query/";
+
+ // String output_parent_dir = parent_dir + "result/";
+ // String outputPath = output_parent_dir + "vbp_query.csv";
+
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ // //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/vbp_query/";
+
+ // String outputPath = output_parent_dir + "vbp_query_greater_new.csv";
+ // String outputPath = output_parent_dir + "vbp_query_less_new.csv";
+ // String outputPath = output_parent_dir + "vbp_query_equal_new.csv";
+ // String outputPath = output_parent_dir + "vbp_query_greater_less.csv";
+ // String outputPath = output_parent_dir + "vbp_query_greater_less_new.csv";
+ // String outputPath = output_parent_dir + "vbp_query_count.csv";
+ String outputPath = output_parent_dir + "vbp_query_count2.csv";
+ // String outputPath = output_parent_dir + "vbp_query_max.csv";
+ // String outputPath = output_parent_dir + "vbp_query_sum.csv";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ HashMap<String, Integer> queryLessRange = new HashMap();
+
+ queryLessRange.put("Bird-migration", 2600000);
+ queryLessRange.put("Bitcoin-price", 170000000);
+ queryLessRange.put("City-temp", 700);
+ queryLessRange.put("Dewpoint-temp", 9600);
+ queryLessRange.put("IR-bio-temp", -200);
+ queryLessRange.put("PM10-dust", 2000);
+ queryLessRange.put("Stocks-DE", 90000);
+ queryLessRange.put("Stocks-UK", 30000);
+ queryLessRange.put("Stocks-USA", 6000);
+ queryLessRange.put("Wind-Speed", 60);
+ queryLessRange.put("Wine-Tasting", 10);
+ queryLessRange.put("Arade4", 12000000);
+ queryLessRange.put("EPM-Education", 300);
+ queryLessRange.put("POI-lat", 1);
+ queryLessRange.put("Gov10", 120000);
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ // repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if (!queryRange.containsKey(datasetName)) {
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ long[] data2_arr = new long[data1.size()];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < data1.size(); i++) {
+ data2_arr[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ // test
+ // for (int i = 0; i < data2_arr.length; i++) {
+ // System.out.print(data2_arr[i] + " ");
+ // }
+ // System.out.println();
+
+ System.out.println(max_decimal);
+ byte[] encoded_result = new byte[data2_arr.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<VBPIndexLong> indexList = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // clear indexList
+ indexList.clear();
+
+ length = VBPIndexLongTest.Encoder(data2_arr, block_size, indexList, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (VBPIndexLong idx : indexList) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ // long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // VBPQueryGreaterTest.Decoder(encoded_result, indexList, queryRange.get(datasetName));
+ // VBPQuerySmallerTest.Decoder(encoded_result, indexList, queryRange.get(datasetName));
+ // VBPQueryEqualTest.Decoder(encoded_result, indexList, queryRange.get(datasetName));
+ // VBPQueryGreaterLessTest.Decoder(encoded_result, indexList, queryRange.get(datasetName), queryLessRange.get(datasetName));
+ // VBPQueryCountTest.Decoder(encoded_result, indexList, queryRange.get(datasetName));
+ VBPQueryCount2Test.Decoder(encoded_result, indexList);
+ // VBPQueryMaxTest.Decoder(encoded_result, indexList);
+ // VBPQuerySumTest.Decoder(encoded_result, indexList);
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+
+ @Test
+ public void testParts() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // // String parent_dir = "D:/encoding-subcolumn/";
+ //
+ String input_parent_dir = parent_dir + "dataset/";
+ //
+ String output_parent_dir = "D:/encoding-subcolumn/result/vbp_query/";
+ // // String output_parent_dir = parent_dir + "result/vbp_query/";
+
+ // String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/subcolumn/";
+ // //"D:/github/xjz17/subcolumn/";
+ // String parent_dir = "D:/encoding-subcolumn/";
+
+ // String input_parent_dir = parent_dir + "dataset/";
+
+ // String output_parent_dir = parent_dir + "result/vbp_query/";
+
+ String outputPath = output_parent_dir + "vbp_query_less_parts_new.csv";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ // repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if (!queryRange.containsKey(datasetName)) {
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ // long[] data2_arr = new long[data1.size()];
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (long) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<VBPIndexLong> indexList1 = new ArrayList<>();
+ ArrayList<VBPIndexLong> indexList2 = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // clear indexList
+ indexList1.clear();
+ indexList2.clear();
+
+ length = VBPIndexLongTest.Encoder(col1_data, block_size, indexList1, encoded_result1);
+
+ length = VBPIndexLongTest.Encoder(col2_data, block_size, indexList2, encoded_result2);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (VBPIndexLong idx : indexList1) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ for (VBPIndexLong idx : indexList2) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ // long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ VBPQuerySmallerPartsTest.Decoder(encoded_result1, encoded_result2, indexList1, indexList2, queryRange.get(datasetName));
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+ @Test
+ public void testMaterialize() throws IOException {
+ String parent_dir = "D:/github/xjz17/subcolumn/";
+ // // String parent_dir = "D:/encoding-subcolumn/";
+ //
+ String input_parent_dir = parent_dir + "dataset/";
+ //
+ String output_parent_dir = "D:/encoding-subcolumn/result/materialization/";
+
+ String outputPath = output_parent_dir + "bitweaving_materialization.csv";
+
+ HashMap<String, Integer> queryRange = new HashMap<>();
+
+ queryRange.put("Bird-migration", 2500000);
+ queryRange.put("Bitcoin-price", 160000000);
+ queryRange.put("City-temp", 480);
+ queryRange.put("Dewpoint-temp", 9500);
+ queryRange.put("IR-bio-temp", -300);
+ queryRange.put("PM10-dust", 1000);
+ queryRange.put("Stocks-DE", 40000);
+ queryRange.put("Stocks-UK", 20000);
+ queryRange.put("Stocks-USA", 5000);
+ queryRange.put("Wind-Speed", 50);
+ queryRange.put("Wine-Tasting", 0);
+ queryRange.put("Arade4", 10000000);
+ queryRange.put("EPM-Education", 200);
+ queryRange.put("POI-lat", 0);
+ queryRange.put("Gov10", 100000);
+
+ int block_size = 512;
+
+ int repeatTime = 100;
+ // repeatTime = 500;
+
+ // repeatTime = 1;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+
+ String[] head = {
+ "Dataset",
+ "Encoding Algorithm",
+ "Decoding Time",
+ "Points",
+ };
+ writer.writeRecord(head);
+
+ File directory = new File(input_parent_dir);
+ // File[] csvFiles = directory.listFiles();
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+
+ for (File file : csvFiles) {
+ String datasetName = extractFileName(file.toString());
+ System.out.println(datasetName);
+ if (!queryRange.containsKey(datasetName)) {
+ continue;
+ }
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Double> data1 = new ArrayList<>();
+
+ int max_decimal = 0;
+ while (loader.readRecord()) {
+ String f_str = loader.getValues()[0];
+ if (f_str.isEmpty()) {
+ continue;
+ }
+ int cur_decimal = getDecimalPrecision(f_str);
+ if (cur_decimal > max_decimal) {
+ max_decimal = cur_decimal;
+ }
+ data1.add(Double.valueOf(f_str));
+ }
+ inputStream.close();
+
+ if (max_decimal > 17) {
+ max_decimal = 17;
+ }
+
+ // long[] data2_arr = new long[data1.size()];
+ int totalSize = data1.size();
+ int halfSize = totalSize / 2;
+
+ long[] col1_data = new long[halfSize];
+ long[] col2_data = new long[halfSize];
+
+ long max_mul = (long) Math.pow(10, max_decimal);
+ for (int i = 0; i < halfSize; i++) {
+ col1_data[i] = (long) (data1.get(i) * max_mul);
+ }
+
+ for (int i = 0; i < halfSize; i++) {
+ col2_data[i] = (long) (data1.get(i + halfSize) * max_mul);
+ }
+
+ System.out.println(max_decimal);
+
+ byte[] encoded_result1 = new byte[col1_data.length * 8];
+ byte[] encoded_result2 = new byte[col2_data.length * 8];
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ int length = 0;
+
+ ArrayList<VBPIndexLong> indexList1 = new ArrayList<>();
+ ArrayList<VBPIndexLong> indexList2 = new ArrayList<>();
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ // clear indexList
+ indexList1.clear();
+ indexList2.clear();
+
+ length = VBPIndexLongTest.Encoder(col1_data, block_size, indexList1, encoded_result1);
+
+ length = VBPIndexLongTest.Encoder(col2_data, block_size, indexList2, encoded_result2);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime);
+ compressed_size += length;
+
+ for (VBPIndexLong idx : indexList1) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ for (VBPIndexLong idx : indexList2) {
+ compressed_size += idx.k * idx.wordsPerPlane * Long.BYTES;
+ }
+
+ double ratioTmp;
+
+ ratioTmp = compressed_size / (double) (data1.size() * Long.BYTES);
+
+ ratio += ratioTmp;
+
+ System.out.println("Decode");
+
+ // long[] data2_arr_decoded = new long[data2_arr.length];
+
+ s = System.nanoTime();
+
+ int[] res1 = new int[data1.size()];
+ int[] res2 = new int[data1.size()];
+ int[] len1 = new int[1];
+ int[] len2 = new int[1];
+
+ int[] result = new int[data1.size()];
+ int[] result_length = new int[1];
+
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ VBPMaterializeTest.Decoder(encoded_result1, indexList1, queryRange.get(datasetName), res1, len1);
+
+ VBPMaterializeTest.Decoder(encoded_result2, indexList2, queryRange.get(datasetName), res2, len2);
+
+ int i = 0, j = 0;
+ int idx = 0;
+ while (i < len1[0] && j < len2[0]) {
+ if (res1[i] == res2[j]) {
+ result[idx] = res1[i];
+ idx++;
+ i++;
+ j++;
+ } else if (res1[i] < res2[j]) {
+ i++;
+ } else {
+ j++;
+ }
+ }
+
+ }
+
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime);
+
+ String[] record = {
+ datasetName,
+ "BitWeaving",
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+
+ writer.close();
+ }
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMaxTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMaxTest.java
new file mode 100644
index 0000000..eec6884
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQueryMaxTest.java
@@ -0,0 +1,104 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQueryMaxTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, long[] result, int[] result_length) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ int max_index = idx.findMaxIndex();
+
+ if (max_index != -1) {
+ result[result_length[0]] = idx.getCode(max_index) + min_value;
+ } else {
+ result[result_length[0]] = Long.MIN_VALUE;
+ }
+
+ // for (int i = 0; i < remainder; i++) {
+ // long value = idx.getCode(i) + min_value;
+ // if (result[0] < value) {
+ // result[0] = value;
+ // }
+ // }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] result = new long[1];
+ result[0] = Long.MIN_VALUE;
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerPartsTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerPartsTest.java
new file mode 100644
index 0000000..f7edc43
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerPartsTest.java
@@ -0,0 +1,129 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQuerySmallerPartsTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void BlockDecoder(byte[] encoded_result1, byte[] encoded_result2, int block_index, int block_size1, int block_size2,
+ int[] encode_pos, ArrayList<VBPIndexLong> indexList1, ArrayList<VBPIndexLong> indexList2, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value1 = bytes2Long(encoded_result1, encode_pos[0], 8);
+ encode_pos[0] += 8;
+
+ int bw1 = bytes2Integer(encoded_result1, encode_pos[0], 4);
+ encode_pos[0] += 4;
+
+ long min_value2 = bytes2Long(encoded_result2, encode_pos[1], 8);
+ encode_pos[1] += 8;
+
+ int bw2 = bytes2Integer(encoded_result2, encode_pos[1], 4);
+ encode_pos[1] += 4;
+
+
+ VBPIndexLong idx1 = indexList1.get(block_index);
+ VBPIndexLong idx2 = indexList2.get(block_index);
+
+ // BitSet bitset_result1 = idx1.select(VBPIndexLong.Op.LT, bound_query_range);
+ // BitSet bitset_result2 = idx2.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ // for (int i = 0; i < bitset_result1.length(); i++) {
+ // if (bitset_result1.get(i) && bitset_result2.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size1);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result1 = idx1.select(VBPIndexLong.Op.LT, bound_query_range);
+ int[] query_result2 = idx2.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ int i = 0, j = 0;
+ while (i < query_result1.length && j < query_result2.length) {
+ if (query_result1[i] == query_result2[j]) {
+ result[result_length[0]] = query_result1[i] + (block_index * block_size1);
+ result_length[0]++;
+ i++;
+ j++;
+ } else if (query_result1[i] < query_result2[j]) {
+ i++;
+ } else {
+ j++;
+ }
+ }
+
+ }
+
+ public static void Decoder(byte[] encoded_result1, byte[] encoded_result2, ArrayList<VBPIndexLong> indexList1, ArrayList<VBPIndexLong> indexList2, int bound_query_range) {
+ int[] encode_pos = new int[2];
+
+ int data_length1 = bytes2Integer(encoded_result1, encode_pos[0], 4);
+ encode_pos[0] += 4;
+
+ int block_size1 = bytes2Integer(encoded_result1, encode_pos[0], 4);
+ encode_pos[0] += 4;
+
+ int num_blocks = data_length1 / block_size1;
+
+ int data_length2 = bytes2Integer(encoded_result2, encode_pos[1], 4);
+ encode_pos[1] += 4;
+
+ int block_size2 = bytes2Integer(encoded_result2, encode_pos[1], 4);
+ encode_pos[1] += 4;
+
+
+ int[] result = new int[data_length1];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ BlockDecoder(encoded_result1, encoded_result2, i, block_size1, block_size2, encode_pos, indexList1, indexList2, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length1 % block_size1;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ BlockDecoder(encoded_result1, encoded_result2, num_blocks, block_size1, block_size2,
+ encode_pos, indexList1, indexList2, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerTest.java
new file mode 100644
index 0000000..1db4747
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySmallerTest.java
@@ -0,0 +1,105 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQuerySmallerTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, int[] result, int[] result_length,
+ int bound_query_range) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // BitSet bitset_result = idx.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ // for (int i = 0; i < bitset_result.length(); i++) {
+ // if (bitset_result.get(i)) {
+ // result[result_length[0]] = i + (block_index * block_size);
+ // result_length[0]++;
+ // }
+ // }
+
+ int[] query_result = idx.select(VBPIndexLong.Op.LT, bound_query_range);
+
+ for (int i = 0; i < query_result.length; i++) {
+ result[result_length[0]] = query_result[i] + (block_index * block_size);
+ result_length[0]++;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList, int bound_query_range) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ int[] result = new int[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length, bound_query_range);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length, bound_query_range);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySortTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySortTest.java
new file mode 100644
index 0000000..e52f8e8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySortTest.java
@@ -0,0 +1,152 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPQuerySortTest {
+
+ private static final int[] BLOCK_SIZES = {32, 64, 128, 256, 512, 1024, 2048, 4096, 8192};
+
+ public static int[] querySortSingleBlockByDecode(
+ byte[] encodedResult, ArrayList<VBPIndexLong> indexList, int targetBlockId) {
+ long[] decoded = VBPIndexLongTest.Decoder(encodedResult, indexList);
+ int blockSize = VBPIndexLongTest.bytes2Integer(encodedResult, 4, 4);
+ int dataLength = decoded.length;
+ int numBlocks = dataLength / blockSize;
+ int remainder = dataLength % blockSize;
+ int totalBlocks = numBlocks + (remainder > 0 ? 1 : 0);
+ if (targetBlockId < 0 || targetBlockId >= totalBlocks) {
+ return new int[0];
+ }
+
+ int count = targetBlockId < numBlocks ? blockSize : remainder;
+ int start = targetBlockId * blockSize;
+ int[] indices = new int[count];
+ for (int i = 0; i < count; i++) {
+ indices[i] = start + i;
+ }
+
+ for (int i = 0; i < count - 1; i++) {
+ int min = i;
+ for (int j = i + 1; j < count; j++) {
+ long v1 = decoded[indices[j]];
+ long v2 = decoded[indices[min]];
+ if (v1 < v2 || (v1 == v2 && indices[j] < indices[min])) {
+ min = j;
+ }
+ }
+ int t = indices[i];
+ indices[i] = indices[min];
+ indices[min] = t;
+ }
+ return indices;
+ }
+
+ @Test
+ public void testDecodeSortSingleBlock() throws IOException {
+ String parentDir = "path/to/your/directory/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/";
+ String outputPath = outputParentDir + "vbp_query_sort_decode.csv";
+
+ int repeatTime = 100;
+ int targetBlockId = 0;
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Block Size",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ System.out.println(datasetName);
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] dataArr = new long[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ dataArr[i] = (long) (data.get(i) * maxMul);
+ }
+
+ for (int blockSize : BLOCK_SIZES) {
+ byte[] encodedResult = new byte[Math.max(16, dataArr.length * 12)];
+ int length = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ ArrayList<VBPIndexLong> tmpIndexList = new ArrayList<>();
+ length = VBPIndexLongTest.Encoder(dataArr, blockSize, tmpIndexList, encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ ArrayList<VBPIndexLong> indexList = new ArrayList<>();
+ length = VBPIndexLongTest.Encoder(dataArr, blockSize, indexList, encodedResult);
+
+ int[] sortedIndices = null;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ sortedIndices = querySortSingleBlockByDecode(encodedResult, indexList, targetBlockId);
+ }
+ end = System.nanoTime();
+ long queryTime = (end - start) / repeatTime;
+ System.out.println(
+ "blockSize="
+ + blockSize
+ + ", sortedCount: "
+ + (sortedIndices == null ? 0 : sortedIndices.length));
+
+ double compressionRatio = length / (double) (Math.max(1, data.size()) * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(blockSize),
+ String.valueOf(encodeTime),
+ String.valueOf(queryTime),
+ String.valueOf(data.size()),
+ String.valueOf(length),
+ String.valueOf(compressionRatio)
+ });
+ }
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySumTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySumTest.java
new file mode 100644
index 0000000..7bdc92c
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPQuerySumTest.java
@@ -0,0 +1,100 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+
+public class VBPQuerySumTest {
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static long bytes2Long(byte[] encoded, int start, int num) {
+ long value = 0;
+
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static int BlockDecoder(byte[] encoded_result, int block_index, int block_size, int remainder,
+ int encode_pos, ArrayList<VBPIndexLong> indexList, long[] result, int[] result_length) {
+
+ long min_value = bytes2Long(encoded_result, encode_pos, 8);
+ encode_pos += 8;
+
+ int bw = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ VBPIndexLong idx = indexList.get(block_index);
+
+ // result[0] += idx.sum2() + (min_value * remainder);
+
+ result[result_length[0]] = idx.sum() + (min_value * remainder);
+ result_length[0]++;
+
+ // for (int i = 0; i < remainder; i++) {
+ // long value = idx.getCode(i) + min_value;
+ // if (result[0] < value) {
+ // result[0] = value;
+ // }
+ // }
+
+ return encode_pos;
+
+ }
+
+ public static void Decoder(byte[] encoded_result, ArrayList<VBPIndexLong> indexList) {
+ int encode_pos = 0;
+
+ int data_length = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int block_size = bytes2Integer(encoded_result, encode_pos, 4);
+ encode_pos += 4;
+
+ int num_blocks = data_length / block_size;
+
+ long[] result = new long[data_length];
+ int[] result_length = new int[1];
+
+ for (int i = 0; i < num_blocks; i++) {
+ encode_pos = BlockDecoder(encoded_result, i, block_size, block_size, encode_pos, indexList, result,
+ result_length);
+ }
+
+ int remainder = data_length % block_size;
+
+ // if (remainder <= 3) {
+ // for (int i = 0; i < remainder; i++) {
+ // data[num_blocks * block_size + i] = bytes2Long(encoded_result, encode_pos,
+ // 8);
+ // encode_pos += 8;
+ // }
+ // } else {
+ encode_pos = BlockDecoder(encoded_result, num_blocks, block_size, remainder,
+ encode_pos, indexList, result, result_length);
+ // }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateAppendTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateAppendTest.java
new file mode 100644
index 0000000..f49a845
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateAppendTest.java
@@ -0,0 +1,131 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateAppendTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_append.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Append-only Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ long maxValue = Long.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ if (origin[i] > maxValue) {
+ maxValue = origin[i];
+ }
+ }
+ if (origin.length == 0) {
+ continue;
+ }
+ long appendValue = (maxValue == Long.MAX_VALUE) ? maxValue : (maxValue + 1);
+ long[] appended = new long[origin.length + 1];
+ System.arraycopy(origin, 0, appended, 0, origin.length);
+ appended[origin.length] = appendValue;
+
+ byte[] encodedResult = new byte[Math.max(16, appended.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+ int tailStart = 8 + numBlocks * 12;
+ int newRemainder = remainder + 1;
+
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ appended, numBlocks, blockSize, newRemainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long appendTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(appendTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateDeleteSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateDeleteSmallerTest.java
new file mode 100644
index 0000000..a9a21f2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateDeleteSmallerTest.java
@@ -0,0 +1,129 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateDeleteSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_delete_smaller.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Delete Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ }
+ if (origin.length <= 1) {
+ continue;
+ }
+ long[] deleted = new long[origin.length - 1];
+ System.arraycopy(origin, 0, deleted, 0, deleted.length);
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+ int tailStart = 8 + numBlocks * 12;
+ int newRemainder = remainder - 1;
+
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ if (newRemainder == 0) {
+ updatedLength = tailStart;
+ continue;
+ }
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ deleted, numBlocks, blockSize, newRemainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long deleteTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(deleteTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertLargerTest.java
new file mode 100644
index 0000000..a2c1873
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertLargerTest.java
@@ -0,0 +1,131 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateInsertLargerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_insert_larger.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ long maxValue = Long.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ if (origin[i] > maxValue) {
+ maxValue = origin[i];
+ }
+ }
+ if (origin.length == 0) {
+ continue;
+ }
+ long insertValue = (maxValue == Long.MAX_VALUE) ? maxValue : (maxValue + 1);
+ long[] inserted = new long[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = insertValue;
+
+ byte[] encodedResult = new byte[Math.max(16, inserted.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+ int tailStart = 8 + numBlocks * 12;
+ int newRemainder = remainder + 1;
+
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ inserted, numBlocks, blockSize, newRemainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long insertTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(insertTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertSmallerTest.java
new file mode 100644
index 0000000..cca5ff2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateInsertSmallerTest.java
@@ -0,0 +1,131 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateInsertSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_insert_smaller.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ long minValue = Long.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ if (origin[i] < minValue) {
+ minValue = origin[i];
+ }
+ }
+ if (origin.length == 0) {
+ continue;
+ }
+ long insertValue = (minValue == Long.MIN_VALUE) ? minValue : (minValue - 1);
+ long[] inserted = new long[origin.length + 1];
+ System.arraycopy(origin, 0, inserted, 0, origin.length);
+ inserted[origin.length] = insertValue;
+
+ byte[] encodedResult = new byte[Math.max(16, inserted.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+ int tailStart = 8 + numBlocks * 12;
+ int newRemainder = remainder + 1;
+
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ inserted, numBlocks, blockSize, newRemainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long insertTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(insertTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateLargerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateLargerTest.java
new file mode 100644
index 0000000..2b4a80f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateLargerTest.java
@@ -0,0 +1,131 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateLargerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_larger.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ long maxValue = Long.MIN_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ if (origin[i] > maxValue) {
+ maxValue = origin[i];
+ }
+ }
+ if (origin.length == 0) {
+ continue;
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+
+ long[] updated = new long[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ int updateIndex = numBlocks * blockSize + remainder - 1;
+ updated[updateIndex] = (maxValue == Long.MAX_VALUE) ? maxValue : (maxValue + 1);
+
+ int tailStart = 8 + numBlocks * 12;
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ updated, numBlocks, blockSize, remainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long updateTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(updateTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateSmallerTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateSmallerTest.java
new file mode 100644
index 0000000..c55d67b
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/VBPUpdateSmallerTest.java
@@ -0,0 +1,131 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+public class VBPUpdateSmallerTest {
+
+ @Test
+ public void test0() throws IOException {
+ String parentDir = "D://github/xjz17/subcolumn/";
+ String inputParentDir = parentDir + "dataset/";
+ String outputParentDir = parentDir + "result/update/";
+ String outputPath = outputParentDir + "vbp_update_smaller.csv";
+ int blockSize = 512;
+ int repeatTime = 200;
+
+ CsvWriter writer = new CsvWriter(outputPath, ',', StandardCharsets.UTF_8);
+ writer.setRecordDelimiter('\n');
+ writer.writeRecord(
+ new String[] {
+ "Dataset",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Insert Time with Sub-column",
+ "Points",
+ "Remaining Points",
+ "Compressed Size",
+ "Compression Ratio"
+ });
+
+ File directory = new File(inputParentDir);
+ File[] csvFiles = directory.listFiles((dir, name) -> name.endsWith(".csv"));
+ if (csvFiles == null) {
+ writer.close();
+ return;
+ }
+
+ for (File file : csvFiles) {
+ String datasetName = VBPIndexLongTest.extractFileName(file.toString());
+ if (datasetName.equals("POI-lon") || datasetName.equals("POI-lat")) {
+ continue;
+ }
+ System.out.println(datasetName);
+
+ InputStream inputStream = Files.newInputStream(file.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Float> data = new ArrayList<>();
+ int maxDecimal = 0;
+ while (loader.readRecord()) {
+ String fStr = loader.getValues()[0];
+ if (fStr.isEmpty()) {
+ continue;
+ }
+ int curDecimal = VBPIndexLongTest.getDecimalPrecision(fStr);
+ if (curDecimal > maxDecimal) {
+ maxDecimal = curDecimal;
+ }
+ data.add(Float.valueOf(fStr));
+ }
+ inputStream.close();
+
+ int maxMul = (int) Math.pow(10, Math.min(maxDecimal, 8));
+ long[] origin = new long[data.size()];
+ long minValue = Long.MAX_VALUE;
+ for (int i = 0; i < data.size(); i++) {
+ origin[i] = (long) (data.get(i) * maxMul);
+ if (origin[i] < minValue) {
+ minValue = origin[i];
+ }
+ }
+ if (origin.length == 0) {
+ continue;
+ }
+
+ byte[] encodedResult = new byte[Math.max(16, origin.length * 12)];
+ int encodedLength = 0;
+ long start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+ }
+ long end = System.nanoTime();
+ long encodeTime = (end - start) / repeatTime;
+ encodedLength = VBPIndexLongTest.Encoder(origin, blockSize, new ArrayList<>(), encodedResult);
+
+ int numBlocks = origin.length / blockSize;
+ int remainder = origin.length % blockSize;
+ if (remainder <= 0) {
+ continue;
+ }
+
+ long[] updated = new long[origin.length];
+ System.arraycopy(origin, 0, updated, 0, origin.length);
+ int updateIndex = numBlocks * blockSize + remainder - 1;
+ updated[updateIndex] = (minValue == Long.MIN_VALUE) ? minValue : (minValue - 1);
+
+ int tailStart = 8 + numBlocks * 12;
+ int updatedLength = encodedLength;
+ start = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime; repeat++) {
+ int encodePos =
+ VBPIndexLongTest.BlockEncoder(
+ updated, numBlocks, blockSize, remainder, tailStart, new ArrayList<>(), encodedResult);
+ updatedLength = encodePos;
+ }
+ end = System.nanoTime();
+ long updateTime = (end - start) / repeatTime;
+
+ double compressionRatio = encodedLength / (double) (origin.length * Long.BYTES);
+ writer.writeRecord(
+ new String[] {
+ datasetName,
+ "VBP",
+ String.valueOf(encodeTime),
+ String.valueOf(updateTime),
+ String.valueOf(origin.length),
+ String.valueOf(remainder),
+ String.valueOf(updatedLength),
+ String.valueOf(compressionRatio)
+ });
+ }
+ writer.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodec.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodec.java
new file mode 100644
index 0000000..acc7b57
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodec.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.elf;
+
+import org.apache.iotdb.tsfile.encoding.elf.compressor.ElfCompressor;
+import org.apache.iotdb.tsfile.encoding.elf.decompressor.ElfDecompressor;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Double-precision ELF encode/decode for {@link
+ * org.apache.iotdb.tsfile.encoding.DatasetEncoderCompressBinRoundtripBenchTest}.
+ *
+ * <p>Ported from {@code org.urbcomp.startdb.compress.elf} (elf@elf project), plain ELF only (not
+ * ElfOnChimp / ElfOnGorilla).
+ */
+public final class ElfDoublePrecisionBenchCodec {
+
+ private ElfDoublePrecisionBenchCodec() {}
+
+ public static byte[] encode(double[] values) {
+ int scratchBytes = Math.max(values.length * 16, 4096);
+ ElfCompressor compressor = new ElfCompressor(scratchBytes);
+ for (double v : values) {
+ compressor.addValue(v);
+ }
+ compressor.close();
+ return compressor.getBytes();
+ }
+
+ public static double[] decode(byte[] encoded) {
+ ElfDecompressor decompressor = new ElfDecompressor(encoded);
+ List<Double> decoded = decompressor.decompress();
+ double[] out = new double[decoded.size()];
+ for (int i = 0; i < decoded.size(); i++) {
+ out[i] = decoded.get(i);
+ }
+ return out;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodecTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodecTest.java
new file mode 100644
index 0000000..91f8a68
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/ElfDoublePrecisionBenchCodecTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.elf;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ElfDoublePrecisionBenchCodecTest {
+
+ @Test
+ public void roundTripSmallSeries() {
+ double[] values = {0.0, 1.25, -3.5, 42.125, 1e-6, 100.0, 200.5};
+ byte[] encoded = ElfDoublePrecisionBenchCodec.encode(values);
+ Assert.assertTrue(encoded.length > 0);
+ double[] decoded = ElfDoublePrecisionBenchCodec.decode(encoded);
+ Assert.assertEquals(values.length, decoded.length);
+ for (int i = 0; i < values.length; i++) {
+ if (Double.isNaN(values[i])) {
+ Assert.assertTrue(Double.isNaN(decoded[i]));
+ } else {
+ Assert.assertEquals(values[i], decoded[i], 0.0);
+ }
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfInputBitStream.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfInputBitStream.java
new file mode 100644
index 0000000..38f28f2
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfInputBitStream.java
@@ -0,0 +1,1594 @@
+package org.apache.iotdb.tsfile.encoding.elf.chimp;
+
+import it.unimi.dsi.bits.Fast;
+import it.unimi.dsi.fastutil.booleans.BooleanIterator;
+import it.unimi.dsi.fastutil.ints.IntIterators;
+import it.unimi.dsi.fastutil.io.BinIO;
+import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
+import it.unimi.dsi.fastutil.io.RepositionableStream;
+import it.unimi.dsi.io.DebugInputBitStream;
+import it.unimi.dsi.io.NullInputStream;
+import it.unimi.dsi.io.OutputBitStream;
+
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.channels.FileChannel;
+
+
+/** Bit-level input stream.
+ *
+ * <P>This class wraps any {@link InputStream} so that you can treat it as
+ * <em>bit</em> stream. Constructors and methods closely resemble those of
+ * {@link InputStream}. Data can be read from such a stream in several ways:
+ * reading a (long) natural number in fixed-width, unary, γ, shifted γ, δ, ζ and (skewed)
+ * Golomb coding, or reading a number of bits that will be stored in a vector of
+ * bytes. There is limited support for {@link #mark(int)}/{@link #reset()}
+ * operations.
+ *
+ * <P>This class can also {@linkplain #ElfInputBitStream(byte[]) wrap a byte
+ * array}; this is much more lightweight than wrapping a {@link
+ * it.unimi.dsi.fastutil.io.FastByteArrayInputStream} wrapping the array. Overflowing the array
+ * will cause an {@link EOFException}.
+ *
+ * <P>Note that when reading using a vector of bytes bits are read in the
+ * stream format (see {@link OutputBitStream}): the first bit is bit 7 of the
+ * first byte, the eighth bit is bit 0 of the first byte, the ninth bit is bit
+ * 7 of the second byte and so on. When reading natural numbers using some coding,
+ * instead, they are stored in the standard way, that is, in the <strong>lower</strong>
+ * bits.
+ *
+ * <P>Additional features:
+ *
+ * <UL>
+ *
+ * <LI>This class provides an internal buffer. By setting a buffer of
+ * length 0 at creation time, you can actually bypass the buffering system:
+ * Note, however, that several classes providing buffering have synchronised
+ * methods, so using a wrapper instead of the internal buffer is likely to lead
+ * to a performance drop.
+ *
+ * <LI>To work around the schizophrenic relationship between streams and random
+ * access files in {@link java.io}, this class provides a {@link #flush()}
+ * method that resets the internal state. At this point, you can safely reposition
+ * the underlying stream and read again afterwards. For instance, this is safe
+ * and will perform as expected:
+ * <PRE>
+ * FileInputStream fis = new FileInputStream(...);
+ * ElfInputBitStream ibs = new ElfInputBitStream(fis);
+ * ... read operations on ibs ...
+ * ibs.flush();
+ * fis.getChannel().position(...);
+ * ... other read operations on ibs ...
+ * </PRE>
+ *
+ * <P>As a commodity, an instance of this class will try to cast the underlying byte
+ * stream to a {@link RepositionableStream} and to fetch by reflection the {@link
+ * FileChannel} underlying the given input stream, in this
+ * order. If either reference can be successfully fetched, you can use
+ * directly the {@link #position(long) position()} method with argument
+ * <code>pos</code> with the same semantics of a {@link #flush()}, followed by
+ * a call to <code>position(pos / 8)</code> (where the latter method belongs
+ * either to the underlying stream or to its underlying file channel), followed
+ * by a {@link #skip(long) skip(pos % 8)}. However, since the reflective checks are quite
+ * heavy they can be disabled using a {@linkplain ElfInputBitStream#ElfInputBitStream(InputStream, boolean) suitable constructor}.
+ *
+ * <li>Finally, this class implements partially the interface of a boolean iterator.
+ * More precisely, {@link #nextBoolean()} will return the same bit as {@link #readBit()},
+ * and also the same exceptions, whereas <em>{@link #hasNext()} will always return true</em>:
+ * you must be prepared to catch a {@link RuntimeException} wrapping an {@link IOException}
+ * in case the file ends. It
+ * is very difficult to implement completely an eager operator using a input-stream
+ * based model.
+ *
+ * </ul>
+ *
+ * <P><STRONG>This class is not synchronised</STRONG>. If multiple threads
+ * access an instance of this class concurrently, they must be synchronised externally.
+ *
+ * @see InputStream
+ * @see it.unimi.dsi.io.OutputBitStream
+ * @author Sebastiano Vigna
+ * @since 0.1
+ */
+
+public class ElfInputBitStream implements BooleanIterator, Flushable, Closeable {
+ private final static boolean DEBUG = false;
+
+ /* Precomputed tables: the i-th entry decodes the stream fragment of 16 bits given by the binary reprentation of i.
+ * The upper 16 bits contain code lengths, the lower 16 bits decoded values. 0 means undecodable. */
+ public static final int[] GAMMA = new int[256 * 256], DELTA = new int[256 * 256], ZETA_3 = new int[256 * 256], SHIFTED_GAMMA = new int[256 * 256];
+
+ static void fillArrayFromResource(final String resource, final int array[]) throws IOException {
+ final String resouceFullPath = "/it/unimi/dsi/io/" + resource;
+ final InputStream ris = ElfInputBitStream.class.getResourceAsStream(resouceFullPath);
+ if (ris == null) throw new IOException("Cannot open resource " + resouceFullPath);
+ final DataInputStream dis = new DataInputStream(new FastBufferedInputStream(ris));
+ BinIO.loadInts(dis, array, 0, array.length);
+ dis.close();
+ assert checkLength(resource, array, resouceFullPath);
+ }
+
+ public static boolean checkLength(final String resource, final int[] array, final String resouceFullPath) {
+ final DataInputStream dis = new DataInputStream(ElfInputBitStream.class.getResourceAsStream(resouceFullPath));
+ final int actualLength = IntIterators.unwrap(BinIO.asIntIterator(dis)).length;
+ assert array.length == actualLength : resource + " is long " + actualLength + " but we think it should rather be " + array.length;
+ try {
+ dis.close();
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ return true;
+ }
+
+ static {
+ /* We load all precomputed arrays from resource files,
+ * to work around the limit on static initialiser code. */
+ try {
+ fillArrayFromResource("gamma.in.16", GAMMA);
+ fillArrayFromResource("delta.in.16", DELTA);
+ fillArrayFromResource("zeta3.in.16", ZETA_3);
+ fillArrayFromResource("shiftedgamma.in.16", SHIFTED_GAMMA);
+ }
+ catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** The default size of the byte buffer in bytes (8Ki). */
+ public static final int DEFAULT_BUFFER_SIZE = 8 * 1024;
+ /** The underlying {@link InputStream}. */
+ protected final InputStream is;
+ /** Whether we should use the byte buffer. */
+// private final boolean noBuffer;
+ /** The cached file channel underlying {@link #is}, if any. */
+ protected final FileChannel fileChannel;
+ /** {@link #is} cast to a positionable stream, if possible. */
+ protected final RepositionableStream repositionableStream;
+ /** True if we are wrapping an array. */
+ protected final boolean wrapping;
+ /** The number of bits actually read from this bit stream. */
+ private long readBits;
+ /** Current bit buffer: the lowest {@link #fill} bits represent the current content (the remaining bits are undefined). */
+ private int current;
+ /** The stream buffer. */
+ protected byte[] buffer;
+ /** Current number of bits in the bit buffer (stored low). */
+ protected int fill;
+ /** Current position in the byte buffer. */
+ protected int pos;
+ /** Current number of bytes available in the byte buffer. */
+ protected int avail;
+ /** Current position of the first byte in the byte buffer. */
+ protected long position;
+
+
+ /** This (non-public) constructor exists just to provide fake initialisation for classes such as {@link DebugInputBitStream}.
+ */
+ protected ElfInputBitStream() {
+ is = null;
+// noBuffer = true;
+ repositionableStream = null;
+ fileChannel = null;
+ wrapping = false;
+ }
+
+ /** Creates a new input bit stream wrapping a given input stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * <p>This constructor performs the reflective tests that are necessary to support {@link #position(long)}.
+ *
+ * @param is the input stream to wrap.
+ */
+ public ElfInputBitStream(final InputStream is) {
+ this(is, true);
+ }
+
+
+ /** Creates a new input bit stream wrapping a given input stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * @param is the input stream to wrap.
+ * @param testForPosition if false, the reflective test that is necessary to support {@link #position(long)}
+ * in case <code>is</code> does not implement {@link RepositionableStream} will not be performed.
+ */
+ public ElfInputBitStream(final InputStream is, final boolean testForPosition) {
+ this(is, DEFAULT_BUFFER_SIZE, testForPosition);
+ }
+
+ /** Creates a new input bit stream wrapping a given input stream with a specified buffer size.
+ *
+ * <p>This constructor performs the reflective tests that are necessary to support {@link #position(long)}.
+ *
+ * @param is the input stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfInputBitStream(final InputStream is, final int bufSize) {
+ this(is, bufSize, true);
+ }
+
+ /** Creates a new input bit stream wrapping a given input stream with a specified buffer size.
+ *
+ * @param is the input stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ * @param testForPosition if false, the reflective test that is necessary to support {@link #position(long)}
+ * in case <code>is</code> does not implement {@link RepositionableStream} will not be performed.
+ */
+ public ElfInputBitStream(final InputStream is, final int bufSize, final boolean testForPosition) {
+ this.is = is;
+ wrapping = false;
+ this.buffer = new byte[bufSize];
+
+ // Cheap test, we do it all the time
+ if (is instanceof RepositionableStream) {
+ repositionableStream = (RepositionableStream)is;
+ fileChannel = null;
+ }
+ else if (testForPosition) {
+ FileChannel fc = null;
+ try {
+ fc = (FileChannel)(is.getClass().getMethod("getChannel")).invoke(is);
+ }
+ catch(final IllegalAccessException e) {}
+ catch(final IllegalArgumentException e) {}
+ catch(final NoSuchMethodException e) {}
+ catch(final InvocationTargetException e) {}
+ catch(final ClassCastException e) {}
+ fileChannel = fc;
+ repositionableStream = null;
+ }
+ else {
+ repositionableStream = null;
+ fileChannel = null;
+ }
+ }
+
+ /** Creates a new input bit stream wrapping a given file input stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param is the file input stream to wrap.
+ */
+ public ElfInputBitStream(final FileInputStream is) {
+ this(is, DEFAULT_BUFFER_SIZE);
+ }
+
+ /** Creates a new input bit stream wrapping a given file input stream with a specified buffer size.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param is the file input stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfInputBitStream(final FileInputStream is, final int bufSize) {
+ this.is = is;
+ wrapping = false;
+ this.buffer = new byte[bufSize];
+ repositionableStream = null;
+ fileChannel = is.getChannel();
+ }
+
+ /** Creates a new input bit stream wrapping a given byte array.
+ *
+ * @param a the byte array to wrap.
+ */
+ public ElfInputBitStream(final byte[] a) {
+ is = NullInputStream.getInstance();
+ repositionableStream = null;
+ fileChannel = null;
+
+// if (a.length > 0) {
+ buffer = a;
+ avail = a.length;
+ wrapping = true;
+// noBuffer = false;
+// }
+// else {
+// // A zero-length buffer is like having no buffer
+// buffer = null;
+// avail = 0;
+// wrapping = false;
+// noBuffer = true;
+// }
+ }
+
+ /** Creates a new input bit stream reading from a file.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param name the name of the file.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfInputBitStream(final String name, final int bufSize) throws FileNotFoundException {
+ this(new FileInputStream(name), bufSize);
+ }
+
+ /** Creates a new input bit stream reading from a file.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param name the name of the file.
+ */
+ public ElfInputBitStream(final String name) throws FileNotFoundException {
+ this(new FileInputStream(name), DEFAULT_BUFFER_SIZE);
+ }
+
+ /** Creates a new input bit stream reading from a file.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param file the file.
+ */
+ public ElfInputBitStream(final File file) throws FileNotFoundException {
+ this(new FileInputStream(file), DEFAULT_BUFFER_SIZE);
+ }
+
+ /** Creates a new input bit stream reading from a file.
+ *
+ * <p>This constructor invokes directly {@link FileInputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param file the file.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfInputBitStream(final File file, final int bufSize) throws FileNotFoundException {
+ this(new FileInputStream(file), bufSize);
+ }
+
+
+ /** Flushes the bit stream. All state information associated with the stream is reset. This
+ * includes bytes prefetched from the stream, bits in the bit buffer and unget'd bits.
+ *
+ * <P>This method is provided so that users of this class can easily wrap repositionable
+ * streams (for instance, file-based streams, which can be repositioned using
+ * the underlying {@link FileChannel}). It is guaranteed that after calling
+ * this method the underlying stream can be repositioned, and that the next read
+ * will draw data from the stream.
+ */
+
+ @Override
+ public void flush() {
+ if (! wrapping) {
+ position += pos;
+ avail = 0;
+ pos = 0;
+ }
+ fill = 0;
+ }
+
+ /** Closes the bit stream. All resources associated with the stream are released.
+ */
+
+ @Override
+ public void close() throws IOException {
+ if (is != null && is != System.in) is.close();
+ buffer = null;
+ }
+
+ /** Returns the number of bits that can be read (or skipped over) from this
+ * bit stream without blocking by the next caller of a method.
+ *
+ * @return the number of bits that can be read from this bit stream without blocking.
+ */
+
+ public long available() throws IOException {
+ return (is.available() + avail) * 8 + fill;
+ }
+
+
+ /** Returns the number of bits read from this bit stream.
+ *
+ * @return the number of bits read so far.
+ */
+ public long readBits() {
+ return readBits;
+ }
+
+ /** Sets the number of bits read from this bit stream.
+ *
+ * <P>This method is provided so that, for instance, the
+ * user can reset via <code>readBits(0)</code> the read-bits count
+ * after a {@link #flush()}.
+ *
+ * @param readBits the new value for the number of bits read so far.
+ */
+ public void readBits(final long readBits) {
+ this.readBits = readBits;
+ }
+
+ /** Reads the next byte from the stream.
+ *
+ * <P>This method takes care of managing the buffering logic
+ * transparently.
+ *
+ * <P>However, this method does <em>not</em> update {@link #readBits}.
+ * The caller should increment {@link #readBits} by 8 at each call, unless
+ * the bit are used to load {@link #current}.
+ */
+
+ private final int read() throws IOException {
+// if (noBuffer) {
+// final int t = is.read();
+// position++;
+// return t;
+// }
+
+ if (avail == 0) {
+ avail = is.read(buffer);
+ position += pos;
+ pos = 0;
+ }
+
+ avail--;
+ return buffer[pos++] & 0xFF;
+ }
+
+ /** Feeds 16 more bits into {@link #current}, assuming that {@link #fill} is less than 16.
+ *
+ * <p>This method will never throw an {@link EOFException}—simply, it will refill less than 16 bits.
+ *
+ * @return {@link #fill}.
+ */
+
+ private final int refill() throws IOException {
+
+// current = current << 16 | (buffer[pos++] & 0xFF) << 8 | buffer[pos++] & 0xFF;
+// return fill += 16;
+
+ if (avail > 1) {
+ // If there is a byte in the buffer, we use it directly.
+ avail -= 2;
+ current = current << 16 | (buffer[pos++] & 0xFF) << 8 | buffer[pos++] & 0xFF;
+ return fill += 16;
+ } else {
+ }
+
+ current = (current << 8) | read();
+ fill += 8;
+ current = (current << 8) | read();
+ fill += 8;
+
+ return fill;
+ }
+
+
+ /** Reads bits from the bit buffer, possibly refilling it.
+ *
+ * <P>This method is the basic mean for extracting bits from the underlying stream.
+ *
+ * <P>You cannot read more than {@link #fill} bits with this method (unless {@link #fill} is 0,
+ * and <code>len</code> is nonzero, in which case the buffer will be refilled for you with 8 bits), and if you
+ * read exactly {@link #fill} bits the buffer will be empty afterwards. In particular,
+ * there will never be 8 bits in the buffer.
+ *
+ * <P>The bit buffer stores its content in the lower {@link #fill} bits. The content
+ * of the remaining bits is undefined.
+ *
+ * <P>This method updates {@link #readBits}.
+ *
+ * @param len the number of bits to read.
+ * @return the bits read (in the <strong>lower</strong> positions).
+ * @throws AssertionError if one tries to read more bits than available in the buffer and assertions are enabled.
+ */
+
+ private final int readFromCurrent(final int len) throws IOException {
+
+ if (fill == 0) {
+ current = read();
+ fill = 8;
+ }
+
+ return current >>> (fill -= len) & (1 << len) - 1;
+ }
+
+ /** Aligns the stream.
+ *
+ * After a call to this function, the stream is byte aligned. Bits that have been
+ * read to align are discarded.
+ */
+
+ public void align() {
+ if ((fill & 7) == 0) return;
+ readBits += fill & 7;
+ fill &= ~7;
+ }
+
+
+
+ /** Reads a sequence of bits.
+ *
+ * Bits will be read in the natural way: the first bit is bit 7 of the
+ * first byte, the eightth bit is bit 0 of the first byte, the ninth bit is
+ * bit 7 of the second byte and so on.
+ *
+ * @param bits an array of bytes to store the result.
+ * @param len the number of bits to read.
+ */
+
+ public void read(final byte[] bits, int len) throws IOException {
+ assert fill < 32 : fill + " >= " + 32;
+
+ if (len <= fill) {
+ if (len <= 8) {
+ bits[0] = (byte)(readFromCurrent(len) << 8 - len);
+ return;
+ }
+ else if (len <= 16){
+ bits[0] = (byte)(readFromCurrent(8));
+ bits[1] = (byte)(readFromCurrent(len - 8) << 16 - len);
+ return;
+ }
+ else if (len <= 24) {
+ bits[0] = (byte)(readFromCurrent(8));
+ bits[1] = (byte)(readFromCurrent(8));
+ bits[2] = (byte)(readFromCurrent(len - 16) << 24 - len);
+ return;
+ }
+ else {
+ bits[0] = (byte)(readFromCurrent(8));
+ bits[1] = (byte)(readFromCurrent(8));
+ bits[2] = (byte)(readFromCurrent(8));
+ bits[3] = (byte)(readFromCurrent(len - 24) << 32 - len);
+ return;
+ }
+ }
+ else {
+ int i, j = 0, b;
+
+ if (fill >= 24) {
+ bits[j++] = (byte)(readFromCurrent(8));
+ bits[j++] = (byte)(readFromCurrent(8));
+ bits[j++] = (byte)(readFromCurrent(8));
+ len -= 24;
+ }
+ else if (fill >= 16) {
+ bits[j++] = (byte)(readFromCurrent(8));
+ bits[j++] = (byte)(readFromCurrent(8));
+ len -= 16;
+ }
+ else if (fill >= 8) {
+ bits[j++] = (byte)(readFromCurrent(8));
+ len -= 8;
+ }
+
+ final int shift = fill;
+
+ if (shift != 0) {
+ bits[j] = (byte)(readFromCurrent(shift) << 8 - shift);
+ len -= shift;
+ i = len >> 3;
+ while(i-- != 0) {
+ b = read();
+ bits[j] |= (b & 0xFF) >>> shift;
+ bits[++j] = (byte)(b << 8 - shift);
+ }
+ }
+ else {
+ i = len >> 3;
+ while(i-- != 0) bits[j++] = (byte)read();
+ }
+
+ readBits += len & ~7;
+
+ len &= 7;
+ if (len != 0) {
+ if (shift == 0) bits[j] = 0; // We must zero the next byte before OR'ing stuff in
+ if (len <= 8 - shift) {
+ bits[j] |= (byte)(readFromCurrent(len) << 8 - shift - len);
+ }
+ else {
+ bits[j] |= (byte)(readFromCurrent(8 - shift));
+ bits[j + 1] = (byte)(readFromCurrent(len + shift - 8) << 16 - shift - len);
+ }
+ }
+ }
+ }
+
+
+
+ /** Reads a bit.
+ *
+ * @return the next bit from the stream.
+ */
+
+ public int readBit() throws IOException {
+ return readFromCurrent(1);
+ }
+
+
+ /** Reads a fixed number of bits into an integer.
+ *
+ * @param len a bit length.
+ * @return an integer whose lower <code>len</code> bits are taken from the stream; the rest is zeroed.
+ * @throws IOException
+ */
+
+ public int readInt(int len) throws IOException {
+ int i, x = 0;
+
+ if (fill < 16) refill();
+ if (len <= fill) return readFromCurrent(len);
+
+ len -= fill;
+ x = readFromCurrent(fill);
+
+ i = len >> 3;
+ while(i-- != 0) x = x << 8 | read();
+ readBits += len & ~7;
+
+ len &= 7;
+
+ return (x << len) | readFromCurrent(len);
+
+ }
+
+
+ /** Reads a fixed number of bits into a long.
+ *
+ * @param len a bit length.
+ * @return a long whose lower <code>len</code> bits are taken from the stream; the rest is zeroed.
+ */
+
+ public long readLong(int len) throws IOException {
+ int i;
+ long x = 0;
+
+ if (fill < 16) refill();
+ if (len <= fill) return readFromCurrent(len);
+
+ len -= fill;
+ x = readFromCurrent(fill);
+
+ i = len >> 3;
+ while(i-- != 0) x = x << 8 | read();
+
+ len &= 7;
+
+ return (x << len) | readFromCurrent(len);
+ }
+
+
+ /** Skips the given number of bits.
+ *
+ * @param n the number of bits to skip.
+ * @return the actual number of skipped bits.
+ */
+
+ public long skip(long n) throws IOException {
+ if (n <= fill) {
+ if (n < 0) throw new IllegalArgumentException("Negative bit skip value: " + n);
+ fill -= n;
+ readBits += n;
+ return n;
+ }
+ else {
+ final long prevReadBits = readBits;
+
+ n -= fill;
+ readBits += fill;
+ fill = 0;
+
+ long nb = n >> 3;
+
+ // TODO: A real evaluation of the usefulness of this block of code
+ if (buffer != null && nb > avail && nb < avail + buffer.length) {
+ /* If we can skip by simply filling the buffer and skipping some bytes,
+ we do it. Usually the next block has already been fetched by a read-ahead logic. */
+ readBits += (avail + 1) << 3;
+ n -= (avail + 1) << 3;
+ nb -= avail + 1;
+ position += pos + avail;
+ pos = avail = 0;
+ read();
+ }
+
+ if (nb <= avail) {
+ // We skip bytes directly inside the buffer.
+ pos += (int)nb;
+ avail -= (int)nb;
+ readBits += n & ~7;
+ }
+ else {
+ // No way, we have to pass the byte skip to the underlying stream.
+ n -= avail << 3;
+ readBits += avail << 3;
+
+ final long toSkip = nb - avail;
+ // ALERT: the semantics of skip is flawed--this should be somehow fixed.
+ final long skipped = is.skip(toSkip);
+ if (skipped < toSkip) throw new IOException("skip() has skipped " + skipped + " instead of " + toSkip + " bytes");
+
+ position += (avail + pos) + skipped;
+ pos = 0;
+ avail = 0;
+
+ readBits += skipped << 3;
+
+ if (skipped != toSkip) return readBits - prevReadBits;
+ }
+
+ final int residual = (int)(n & 7);
+ if (residual != 0) {
+ current = read();
+ fill = 8 - residual;
+ readBits += residual;
+ }
+ return readBits - prevReadBits;
+ }
+ }
+
+ /** Sets this stream bit position, if it is based on a {@link RepositionableStream} or on a {@link FileChannel}.
+ *
+ * <P>Given an underlying stream that implements {@link
+ * RepositionableStream} or that can provide a {@link
+ * FileChannel} via the <code>getChannel()</code> method,
+ * a call to this method has the same semantics of a {@link #flush()},
+ * followed by a call to {@link
+ * FileChannel#position(long) position(position / 8)} on
+ * the byte stream, followed by a {@link #skip(long) skip(position % 8)}.
+ *
+ * <p>Note that this method does <em>not</em> change the value returned by {@link #readBits()}.
+ *
+ * @param position the new position expressed as a bit offset.
+ * @throws UnsupportedOperationException if the underlying byte stream does not implement
+ * {@link RepositionableStream} or if the channel it returns is not a {@link FileChannel}.
+ * @see FileChannel#position(long)
+ */
+
+ public void position(final long position) throws IOException {
+ if (DEBUG) System.err.println(this + ".position(" + position + ")");
+
+ if (position < 0) throw new IllegalArgumentException("Illegal position: " + position);
+
+ final long bitDelta = ((this.position + pos) << 3) - position;
+ if (bitDelta >= 0 && bitDelta <= fill) {
+ if (DEBUG) System.err.println("Bit positioning... position: " + position + " this.position: " + this.position + " pos: " + pos + " bitDelta: " + bitDelta + " fill: " + fill);
+ fill = (int)bitDelta;
+ //System.err.println("Post: " + position + " fill: " + fill);
+ return;
+ }
+
+ final long delta = (position >> 3) - (this.position + pos);
+
+ if (DEBUG) System.err.println(this + ".position(" + position + "); curr: " + this.position + " delta: " + delta + " pos: " + pos + " avail: " + avail);
+
+ if (delta <= avail && delta >= - pos) {
+ // We can reposition just by moving into the buffer.
+ avail -= delta;
+ pos += delta;
+ fill = 0;
+ if (DEBUG) System.err.println(this + ": moved internally; pos: " + pos + " avail: " + avail);
+ }
+ else if (repositionableStream != null) {
+ flush();
+ repositionableStream.position(this.position = position >> 3);
+ }
+ else if (fileChannel != null) {
+ flush();
+ fileChannel.position(this.position = position >> 3);
+ }
+ else {
+ if (wrapping) throw new UnsupportedOperationException("Illegal position: " + position);
+ throw new UnsupportedOperationException("position() can only be called if the underlying byte stream implements the RepositionableStream interface or if the getChannel() method of the underlying byte stream exists and returns a FileChannel");
+ }
+
+ final int residual = (int)(position & 7);
+
+ if (DEBUG) System.err.println(this + ": residual=" + residual);
+
+ if (residual != 0) {
+ current = read();
+ fill = 8 - residual;
+ }
+ }
+
+ /** Returns this stream bit position.
+ * @return this stream bit position. */
+ public long position() {
+ return ((this.position + pos) << 3) - fill;
+ }
+
+ /** Tests if this stream supports the {@link #mark(int)} and {@link #reset()} methods.
+ *
+ * <P>This method will just delegate the test to the underlying {@link InputStream}.
+ * @return whether this stream supports {@link #mark(int)}/{@link #reset()}.
+ */
+
+ public boolean markSupported() {
+ return is.markSupported();
+ }
+
+ /** Marks the current position in this input stream. A subsequent call to
+ * the {@link #reset()} method repositions this stream at the last marked position so
+ * that subsequent reads re-read the same bits.
+ *
+ * <P>This method will just delegate the mark to the underlying {@link InputStream}.
+ * Moreover, it will throw an exception if you try to mark outsite byte boundaries.
+ *
+ * @param readLimit the maximum limit of bytes that can be read before the mark position becomes invalid.
+ * @throws IOException if you try to mark outside byte boundaries.
+ */
+
+ public void mark(final int readLimit) throws IOException {
+ if (fill != 0) throw new IOException("You cannot mark a bit stream outside of byte boundaries.");
+ is.mark(readLimit);
+ }
+
+ /** Repositions this bit stream to the position at the time the {@link #mark(int)} method was last called.
+ *
+ * <P>This method will just {@link #flush() flush the stream} and delegate
+ * the reset to the underlying {@link InputStream}.
+ */
+
+ public void reset() throws IOException {
+ flush();
+ is.reset();
+ }
+
+ /** Reads a natural number in unary coding.
+ *
+ * @return the next unary-encoded natural number.
+ * @see OutputBitStream#writeUnary(int)
+ */
+
+ public int readUnary() throws IOException {
+ assert fill < 32 : fill + " >= " + 32;
+ int x;
+
+ if (fill < 16) refill();
+ x = Integer.numberOfLeadingZeros(current << (32 - fill));
+ if (x < fill) { // This works also when fill = 0
+ readBits += x + 1;
+ fill -= x + 1;
+ return x;
+ }
+
+ x = fill;
+ while((current = read()) == 0) x += 8;
+ x += 7 - (fill = 31 - Integer.numberOfLeadingZeros(current));
+ readBits += x + 1;
+ return x;
+ }
+
+ /** Reads a long natural number in unary coding.
+ *
+ * Note that by unary coding we mean that 1 encodes 0, 01 encodes 1 and so on.
+ *
+ * @return the next unary-encoded long natural number.
+ * @see OutputBitStream#writeUnary(int)
+ */
+
+ public long readLongUnary() throws IOException {
+ assert fill < 32 : fill + " >= " + 32;
+
+ if (fill < 16) refill();
+ long x = Integer.numberOfLeadingZeros(current << (32 - fill));
+ if (x < fill) { // This works also when fill = 0
+ readBits += x + 1;
+ fill -= x + 1;
+ return x;
+ }
+
+ x = fill;
+ while((current = read()) == 0) x += 8;
+ x += 7 - (fill = 31 - Integer.numberOfLeadingZeros(current));
+ readBits += x + 1;
+ return x;
+ }
+
+ /** Reads a natural number in γ coding.
+ *
+ * @return the next γ-encoded natural number.
+ * @see OutputBitStream#writeGamma(int)
+ * @see #skipGammas(int)
+ */
+
+ public int readGamma() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readUnary();
+ return ((1 << msb) | readInt(msb)) - 1;
+ }
+
+ /** Reads a long natural number in γ coding.
+ *
+ * @return the next γ-encoded long natural number.
+ * @see OutputBitStream#writeGamma(int)
+ * @see #skipGammas(int)
+ */
+
+ public long readLongGamma() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readUnary();
+ return ((1L << msb) | readLong(msb)) - 1;
+ }
+
+ /** Skips a given amount of γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readGamma()} or {@link #readLongGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of γ-coded natural numbers to be skipped.
+ * @see #readGamma()
+ */
+
+ public void skipGammas(long n) throws IOException {
+ int preComp;
+ while(n-- != 0) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = GAMMA[current >> (fill - 16) & 0xFFFF] >> 16) != 0) {
+ readBits += preComp;
+ fill -= preComp;
+ continue;
+ }
+
+ skip((long)readUnary());
+ }
+ }
+
+ /** Skips a given amount of γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readGamma()} or {@link #readLongGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of γ-coded natural numbers to be skipped.
+ * @see #readGamma()
+ */
+ public void skipGammas(final int n) throws IOException {
+ skipGammas((long)n);
+ }
+
+ /** Reads a given amount of γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced.
+ *
+ * @param a an array of at least <code>count</code> integers where the result
+ * will be written starting at the first position.
+ * @param count the number of γ-coded natural numbers to be read.
+ * @see #readGamma()
+ */
+
+ public void readGammas(final int[] a, final int count) throws IOException {
+ int preComp, msb;
+ for(int i = 0; i < count; i++) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ a[i] = preComp & 0xFFFF;
+ continue;
+ }
+
+ a[i] = ((1 << (msb = readUnary())) | readInt(msb)) - 1;
+ }
+ }
+
+ /** Reads a natural number in shifted γ coding.
+ *
+ * @return the next shifted-γ–encoded natural number.
+ * @see OutputBitStream#writeShiftedGamma(int)
+ * @see #skipShiftedGammas(int)
+ */
+
+ public int readShiftedGamma() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = SHIFTED_GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readUnary() - 1;
+ return msb == -1 ? 0 : ((1 << msb) | readInt(msb));
+ }
+
+ /** Reads a natural number in shifted γ coding.
+ *
+ * @return the next shifted-γ–encoded natural number.
+ * @see OutputBitStream#writeShiftedGamma(int)
+ * @see #skipShiftedGammas(int)
+ */
+
+ public long readLongShiftedGamma() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = SHIFTED_GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readUnary() - 1;
+ return msb == -1 ? 0 : ((1L << msb) | readLong(msb));
+ }
+
+ /** Skips a given amount of shifted-γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readShiftedGamma()} or {@link #readLongShiftedGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of shifted-γ-coded natural numbers to be skipped.
+ * @see #readShiftedGamma()
+ */
+
+ public void skipShiftedGammas(long n) throws IOException {
+ int preComp;
+ while(n-- != 0) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = SHIFTED_GAMMA[current >> (fill - 16) & 0xFFFF] >> 16) != 0) {
+ readBits += preComp;
+ fill -= preComp;
+ continue;
+ }
+
+ final long msb = readUnary() - 1;
+ if (msb > 0) skip(msb);
+ }
+ }
+
+ /** Skips a given amount of shifted-γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readShiftedGamma()} or {@link #readLongShiftedGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of shifted-γ-coded natural numbers to be skipped.
+ * @see #readShiftedGamma()
+ */
+
+ public void skipShiftedGammas(final int n) throws IOException {
+ skipShiftedGammas((long)n);
+ }
+
+
+ /** Reads a given amount of shifted-γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readShiftedGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced.
+ *
+ * @param a an array of at least <code>count</code> integers where the result
+ * will be written starting at the first position.
+ * @param count the number of shifted-γ-coded natural numbers to be read.
+ * @see #readShiftedGamma()
+ */
+
+ public void readShiftedGammas(final int[] a, final int count) throws IOException {
+ int preComp, msb;
+ for(int i = 0; i < count; i++) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = SHIFTED_GAMMA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ a[i] = preComp & 0xFFFF;
+ continue;
+ }
+
+ msb = readUnary() - 1;
+ a[i] = msb == -1 ? 0 : ((1 << msb) | readInt(msb));
+ }
+ }
+
+ /** Reads a natural number in δ coding.
+ *
+ * @return the next δ-encoded natural number.
+ * @see OutputBitStream#writeDelta(int)
+ * @see #skipDeltas(int)
+ */
+
+ public int readDelta() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = DELTA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readGamma();
+ return ((1 << msb) | readInt(msb)) - 1;
+ }
+
+
+ /** Reads a long natural number in δ coding.
+ *
+ * @return the next δ-encoded long natural number.
+ * @see OutputBitStream#writeDelta(int)
+ * @see #skipDeltas(int)
+ */
+
+ public long readLongDelta() throws IOException {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = DELTA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+
+ final int msb = readGamma();
+ return ((1L << msb) | readLong(msb)) - 1;
+ }
+
+
+ /** Skips a given amount of δ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readDelta()} or {@link #readLongDelta()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of δ-coded natural numbers to be skipped.
+ * @see #readDelta()
+ */
+
+ public void skipDeltas(long n) throws IOException {
+ int preComp;
+ while(n-- != 0) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = DELTA[current >> (fill - 16) & 0xFFFF] >> 16) != 0) {
+ readBits += preComp;
+ fill -= preComp;
+ continue;
+ }
+
+ skip((long)readGamma());
+ }
+ }
+
+ /** Skips a given amount of δ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readDelta()} or {@link #readLongDelta()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ * @param n the number of δ-coded natural numbers to be skipped.
+ * @see #readDelta()
+ */
+
+ public void skipDeltas(final int n) throws IOException {
+ skipDeltas((long)n);
+ }
+
+ /** Reads a given amount of δ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readDelta()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced.
+ *
+ * @param a an array of at least <code>count</code> integers where the result
+ * will be written starting at the first position.
+ * @param count the number of δ-coded natural numbers to be read.
+ * @see #readDelta()
+ */
+
+ public void readDeltas(final int[] a, final int count) throws IOException {
+ int preComp, msb;
+ for(int i = 0; i < count; i++) {
+ if ((fill >= 16 || refill() >= 16) && (preComp = DELTA[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ a[i] = preComp & 0xFFFF;
+ continue;
+ }
+
+ a[i] = ((1 << (msb = readGamma())) | readInt(msb)) - 1;
+ }
+ }
+
+ /** Reads a natural number in a limited range using a minimal binary coding.
+ *
+ * @param b a strict upper bound.
+ * @return the next minimally binary encoded natural number.
+ * @throws IllegalArgumentException if you try to read a negative number or use a nonpositive base.
+ * @see OutputBitStream#writeMinimalBinary(int, int)
+ */
+
+ public int readMinimalBinary(final int b) throws IOException {
+ return readMinimalBinary(b, Fast.mostSignificantBit(b));
+ }
+
+ /** Reads a natural number in a limited range using a minimal binary coding.
+ *
+ * This method is faster than {@link #readMinimalBinary(int)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param b a strict upper bound.
+ * @param log2b the floor of the base-2 logarithm of the bound.
+ * @return the next minimally binary encoded natural number.
+ * @throws IllegalArgumentException if you try to read a negative number or use a nonpositive base.
+ * @see OutputBitStream#writeMinimalBinary(int, int)
+ */
+
+ public int readMinimalBinary(final int b, final int log2b) throws IOException {
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+
+ final int m = (1 << log2b + 1) - b;
+ final int x = readInt(log2b);
+
+ if (x < m) return x;
+ else return ((x << 1) + readBit() - m);
+ }
+
+ /** Reads a long natural number in a limited range using a minimal binary coding.
+ *
+ * @param b a strict upper bound.
+ * @return the next minimally binary encoded long natural number.
+ * @throws IllegalArgumentException if you try to read a negative number or use a nonpositive base.
+ * @see OutputBitStream#writeMinimalBinary(int, int)
+ */
+
+ public long readLongMinimalBinary(final long b) throws IOException {
+ return readLongMinimalBinary(b, Fast.mostSignificantBit(b));
+ }
+
+
+ /** Reads a long natural number in a limited range using a minimal binary coding.
+ *
+ * This method is faster than {@link #readLongMinimalBinary(long)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param b a strict upper bound.
+ * @param log2b the floor of the base-2 logarithm of the bound.
+ * @return the next minimally binary encoded long natural number.
+ * @throws IllegalArgumentException if you try to read a negative number or use a nonpositive base.
+ * @see OutputBitStream#writeMinimalBinary(int, int)
+ */
+
+ public long readLongMinimalBinary(final long b, final int log2b) throws IOException {
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+
+ final long m = (1L << log2b + 1) - b;
+ final long x = readLong(log2b);
+
+ if (x < m) return x;
+ else return ((x << 1) + readBit() - m);
+ }
+
+ /** Reads a natural number in Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @return the next Golomb-encoded natural number.
+ * @throws IllegalArgumentException if you use a nonpositive modulus.
+ * @see OutputBitStream#writeGolomb(int, int)
+ */
+
+ public int readGolomb(final int b) throws IOException {
+ return readGolomb(b, Fast.mostSignificantBit(b));
+ }
+
+ /** Reads a natural number in Golomb coding.
+ *
+ * This method is faster than {@link #readGolomb(int)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @param log2b the floor of the base-2 logarithm of the coding modulus.
+ * @return the next Golomb-encoded natural number.
+ * @throws IllegalArgumentException if you use a nonpositive modulus.
+ * @see OutputBitStream#writeGolomb(int, int)
+ */
+
+ public int readGolomb(final int b, final int log2b) throws IOException {
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) return 0;
+
+ return readUnary() * b + readMinimalBinary(b, log2b);
+ }
+
+
+
+ /** Reads a long natural number in Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @return the next Golomb-encoded long natural number.
+ * @throws IllegalArgumentException if you use a nonpositive modulus.
+ * @see OutputBitStream#writeGolomb(int, int)
+ */
+
+ public long readLongGolomb(final long b) throws IOException {
+ return readLongGolomb(b, Fast.mostSignificantBit(b));
+ }
+
+ /** Reads a long natural number in Golomb coding.
+ *
+ * This method is faster than {@link #readLongGolomb(long)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @param log2b the floor of the base-2 logarithm of the coding modulus.
+ * @return the next Golomb-encoded long natural number.
+ * @throws IllegalArgumentException if you use a nonpositive modulus.
+ * @see OutputBitStream#writeGolomb(int, int)
+ */
+
+ public long readLongGolomb(final long b, final int log2b) throws IOException {
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) return 0;
+
+ return readUnary() * b + readLongMinimalBinary(b, log2b);
+ }
+
+ /** Reads a natural number in skewed Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @return the next skewed Golomb-encoded natural number.
+ * @throws IllegalArgumentException if you use a negative modulus.
+ * @see OutputBitStream#writeSkewedGolomb(int, int)
+ */
+
+ public int readSkewedGolomb(final int b) throws IOException {
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) return 0;
+
+ final int M = ((1 << readUnary() + 1) - 1) * b;
+ final int m = (M / (2 * b)) * b;
+ return m + readMinimalBinary(M - m);
+ }
+
+
+ /** Reads a long natural number in skewed Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * nothing will be read, and 0 will be returned.
+ *
+ * @param b the modulus for the coding.
+ * @return the next skewed Golomb-encoded long natural number.
+ * @throws IllegalArgumentException if you use a negative modulus.
+ * @see OutputBitStream#writeSkewedGolomb(int, int)
+ */
+
+ public long readLongSkewedGolomb(final long b) throws IOException {
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) return 0;
+
+ final long M = ((1L << readUnary() + 1) - 1) * b;
+ final long m = (M / (2 * b)) * b;
+ return m + readLongMinimalBinary(M - m);
+ }
+
+
+ /** Reads a natural number in ζ coding.
+ *
+ * @param k the shrinking factor.
+ * @return the next ζ-encoded natural number.
+ * @throws IllegalArgumentException if you use a nonpositive shrinking factor.
+ * @see OutputBitStream#writeZeta(int, int)
+ */
+
+ public int readZeta(final int k) throws IOException {
+ if (k < 1) throw new IllegalArgumentException("The shrinking factor " + k + " is not positive");
+
+ if (k == 3) {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = ZETA_3[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+ }
+
+ final int h = readUnary();
+ final int left = 1 << h * k;
+ final int m = readInt(h * k + k - 1);
+ if (m < left) return m + left - 1;
+ return (m << 1) + readBit() - 1;
+ }
+
+ /** Reads a long natural number in ζ coding.
+ *
+ * @param k the shrinking factor.
+ * @return the next ζ-encoded long natural number.
+ * @throws IllegalArgumentException if you use a nonpositive shrinking factor.
+ * @see OutputBitStream#writeZeta(int, int)
+ */
+
+ public long readLongZeta(final int k) throws IOException {
+ if (k < 1) throw new IllegalArgumentException("The shrinking factor " + k + " is not positive");
+
+ if (k == 3) {
+ int preComp;
+ if ((fill >= 16 || refill() >= 16) && (preComp = ZETA_3[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ return preComp & 0xFFFF;
+ }
+ }
+
+ final int h = readUnary();
+ final long left = 1L << h * k;
+ final long m = readLong(h * k + k - 1);
+ if (m < left) return m + left - 1;
+ return (m << 1) + readBit() - 1;
+ }
+
+
+ /** Skips a given amount of ζ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readZeta(int)} or {@link #readLongZeta(int)}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ *
+ * @param k the shrinking factor.
+ * @param n the number of ζ-coded natural numbers to be skipped.
+ * @see #readZeta(int)
+ */
+
+ public void skipZetas(final int k, long n) throws IOException {
+ int h, preComp;
+
+ while(n-- != 0) {
+ if (k == 3 && (fill >= 16 || refill() >= 16) && (preComp = ZETA_3[current >> (fill - 16) & 0xFFFF] >> 16) != 0) {
+ readBits += preComp;
+ fill -= preComp;
+ continue;
+ }
+
+ h = readUnary();
+ if (readInt(h * k + k - 1) >= 1 << h * k) skip(1L);
+ }
+ }
+
+ /** Skips a given amount of ζ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readZeta(int)} or {@link #readLongZeta(int)}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced, and the result is discarded, so
+ * {@link #skip(long)} can be invoked instead of more specific decoding methods.
+ *
+ *
+ * @param k the shrinking factor.
+ * @param n the number of ζ-coded natural numbers to be skipped.
+ * @see #readZeta(int)
+ */
+
+ public void skipZetas(final int k, final int n) throws IOException {
+ skipZetas(k, (long)n);
+ }
+
+ /** Reads a given amount of γ-coded natural numbers.
+ *
+ * <p>This method should be significantly quicker than iterating <code>n</code> times on
+ * {@link #readGamma()}, as precomputed tables are used directly,
+ * so the number of method calls is greatly reduced.
+ *
+ * @param k the shrinking factor.
+ * @param a an array of at least <code>count</code> integers where the result
+ * will be written starting at the first position.
+ * @param count the number of ζ-coded natural numbers to be read.
+ * @see #readGamma()
+ */
+
+ public void readZetas(final int k, final int[] a, final int count) throws IOException {
+ int h, left, m;
+ int preComp;
+
+ for(int i = 0; i < count; i++) {
+ if (k == 3 && (fill >= 16 || refill() >= 16) && (preComp = ZETA_3[current >> (fill - 16) & 0xFFFF]) != 0) {
+ readBits += preComp >> 16;
+ fill -= preComp >> 16;
+ a[i] = preComp & 0xFFFF;
+ continue;
+ }
+
+ h = readUnary();
+ left = 1 << h * k;
+ m = readInt(h * k + k - 1);
+ a[i] = m < left ? m + left - 1 : (m << 1) + readBit() - 1;
+ }
+ }
+
+ /** Reads a natural number in variable-length nibble coding.
+ *
+ * @return the next variable-length nibble-encoded natural number.
+ * @see OutputBitStream#writeNibble(int)
+ */
+
+ public int readNibble() throws IOException {
+ int b;
+ int x = 0;
+
+ do {
+ x <<= 3;
+ b = readBit();
+ x |= readInt(3);
+ } while(b == 0);
+
+ return x;
+ }
+
+ /** Reads a long natural number in variable-length nibble coding.
+ *
+ * @return the next variable-length nibble-encoded long natural number.
+ * @see OutputBitStream#writeNibble(int)
+ */
+
+ public long readLongNibble() throws IOException {
+ int b;
+ long x = 0;
+
+ do {
+ x <<= 3;
+ b = readBit();
+ x |= readInt(3);
+ } while(b == 0);
+
+ return x;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public boolean nextBoolean() {
+ try {
+ return readBit() != 0;
+ }
+ catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Skips over the given number of bits.
+ *
+ * @param n the number of bits to skip.
+ * @return the number of bits actually skipped.
+ * @deprecated This method is simply an expensive, try/catch-surrounded version
+ * of {@link #skip(long)} that is made necessary by the interface
+ * by {@link BooleanIterator}.
+ */
+ @Override
+ @Deprecated
+ public int skip(final int n) {
+ try {
+ return (int)skip((long)n);
+ }
+ catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** Copies a given number of bits from this input bit stream into a given output bit stream.
+ *
+ * @param obs an output bit stream.
+ * @param length the number of bits to copy.
+ * @throws EOFException if there are not enough bits to copy.
+ */
+ public void copyTo(final OutputBitStream obs, long length) throws IOException {
+ final byte[] buffer = new byte[64 * 1024];
+ while(length > 0) {
+ final int toRead = (int)Math.min(length, buffer.length * Byte.SIZE);
+ read(buffer, toRead);
+ obs.write(buffer, 0, toRead);
+ length -= toRead;
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfOutputBitStream.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfOutputBitStream.java
new file mode 100644
index 0000000..4b9f11a
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/chimp/ElfOutputBitStream.java
@@ -0,0 +1,1304 @@
+package org.apache.iotdb.tsfile.encoding.elf.chimp;
+
+import it.unimi.dsi.bits.Fast;
+import it.unimi.dsi.fastutil.booleans.BooleanIterator;
+import it.unimi.dsi.fastutil.io.RepositionableStream;
+
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.channels.FileChannel;
+
+
+/** Bit-level output stream.
+ *
+ * <P>This class wraps any {@link OutputStream} so that you can treat it as
+ * <em>bit</em> stream. Constructors and methods closely resemble those of
+ * {@link OutputStream}. Data can be added to such a stream in several ways:
+ * writing an integer or long in fixed-width, unary, γ, δ, ζ and Golomb
+ * coding, or providing a vector of bytes.
+ *
+ * <P>This class can also {@linkplain #ElfOutputBitStream(byte[]) wrap a byte
+ * array}; this is much more lightweight than wrapping a {@link
+ * it.unimi.dsi.fastutil.io.FastByteArrayOutputStream} wrapping the array, but overflowing the array
+ * will cause an {@link IOException}.
+ *
+ * <P>Note that when writing using a vector of bytes bits are written in the natural
+ * way: the first bit is bit 7 of the first byte, the eighth bit is bit 0 of
+ * the first byte, the ninth bit is bit 7 of the second byte and so on. When
+ * writing integers using some coding, instead, the <em>lower</em> bits are considered
+ * for coding (in the fixed-width case, the given number of bits, otherwise
+ * the lower bits starting from the most significant one).
+ *
+ * <h2>The bit stream format</h2>
+ *
+ * <P>The bit streams written by this class are <em>big endian</em>. That is,
+ * the first bit of the stream is bit 7 of the first byte, the eightth bit
+ * is bit 0 of the first byte, the ninth bit is bit 7 of the second byte and so on.
+ *
+ * <P>Blocks of bits (such as coded integers) are written <em>starting from the
+ * most significant bit</em>. In other words, if you take the first bytes of a stream
+ * and print them in binary you will see exactly the sequence of bits you have
+ * written. In particular, if you write 32-bit integers you will get a stream
+ * which is identical to the one produced by a {@link java.io.DataOutput}.
+ *
+ * <P>Additional features:
+ *
+ * <ul>
+ *
+ * <LI>This class provides an internal buffer. By setting a buffer of
+ * length 0 at creation time, you can actually bypass the buffering system:
+ * Note, however, that several classes providing buffering have synchronised
+ * methods, so using a wrapper instead of the internal buffer is likely to lead
+ * to a performance drop.
+ *
+ * <LI>To work around the schizophrenic relationship between streams and random
+ * access files in {@link java.io}, this class provides a {@link #flush()}
+ * method that byte-aligns the streams, flushes to the underlying byte stream
+ * all data and resets the internal state. At this point, you can safely reposition
+ * the underlying stream and write again afterwards. For instance, this is safe
+ * and will perform as expected:
+ * <PRE>
+ * FileOutputStream fos = new FileOutputStream(...);
+ * ElfOutputBitStream obs = new ElfOutputBitStream(fos);
+ * ... write operations on obs ...
+ * obs.flush();
+ * fos.getChannel().position(...);
+ * ... other write operations on obs ...
+ * </PRE>
+ *
+ * <P>As a commodity, an instance of this class will try to cast the underlying
+ * byte stream to a {@link RepositionableStream} and to fetch by reflection the
+ * {@link FileChannel} underlying the given output stream, in
+ * this order. If either reference can be successfully fetched, you can use
+ * directly the {@link #position(long) position()} method with argument
+ * <code>pos</code> with the same semantics of a {@link #flush()}, followed by
+ * a call to <code>position(pos / 8)</code> (where the latter method belongs
+ * either to the underlying stream or to its underlying file channel). The
+ * specified position must be byte aligned, as there is no clean way of reading
+ * a fraction of a byte with the current APIs. However, since the reflective checks are quite
+ * heavy they can be disabled using a {@linkplain ElfOutputBitStream#ElfOutputBitStream(OutputStream, boolean) suitable constructor}.
+ *
+ * </ul>
+ *
+ * <P><STRONG>This class is not synchronised</STRONG>. If multiple threads
+ * access an instance of this class concurrently, they must be synchronised externally.
+ *
+ * @see OutputStream
+ * @see it.unimi.dsi.io.InputBitStream
+ * @author Sebastiano Vigna
+ * @since 0.1
+ */
+
+public class ElfOutputBitStream implements Flushable, Closeable {
+
+ public static final int MAX_PRECOMPUTED = 4096;
+
+ private final static boolean DEBUG = false;
+
+ /* Precomputed tables: the lower 24 bits contain the (right-aligned) code,
+ * the upper 8 bits contain the code length. */
+ public static final int[] GAMMA = new int[MAX_PRECOMPUTED], DELTA = new int[MAX_PRECOMPUTED], ZETA_3 = new int[MAX_PRECOMPUTED],
+ SHIFTED_GAMMA = new int[MAX_PRECOMPUTED];
+
+ static {
+ /* We load all precomputed arrays from resource files,
+ * to work around the limit on static initialiser code. */
+ try {
+ ElfInputBitStream.fillArrayFromResource("gamma.out.12", GAMMA);
+ ElfInputBitStream.fillArrayFromResource("delta.out.12", DELTA);
+ ElfInputBitStream.fillArrayFromResource("zeta3.out.12", ZETA_3);
+ ElfInputBitStream.fillArrayFromResource("shiftedgamma.out.12", SHIFTED_GAMMA);
+ }
+ catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /** The default size of the byte buffer in bytes (16Ki). */
+ public static final int DEFAULT_BUFFER_SIZE = 16 * 1024;
+ /** The underlying {@link OutputStream}. */
+ protected final OutputStream os;
+ /** The number of bits written to this bit stream. */
+ private long writtenBits;
+ /** Current bit buffer. */
+ private int current;
+ /** The stream buffer. */
+ protected byte[] buffer;
+ /** Current number of free bits in the bit buffer (the bits in the buffer are stored high). */
+ protected int free;
+ /** Current position in the byte buffer. */
+ protected int pos;
+ /** Current position of the underlying output stream. */
+ protected long position;
+ /** Current number of bytes available in the byte buffer. */
+ protected int avail;
+ /** Size of the small buffer for temporary usage. */
+ final static int TEMP_BUFFER_SIZE = 128;
+ /** The cached file channel underlying {@link #os}. */
+ protected final FileChannel fileChannel;
+ /** {@link #os} cast to a positionable stream. */
+ protected final RepositionableStream repositionableStream;
+ /** True if we are wrapping an array. */
+ protected final boolean wrapping;
+
+
+ /** This (non-public) constructor exists just to provide fake initialisation for classes such as {@link DebugOutputBitStream}.
+ */
+ protected ElfOutputBitStream() {
+ os = null;
+ fileChannel = null;
+ repositionableStream = null;
+ wrapping = false;
+ }
+
+
+ /** Creates a new output bit stream wrapping a given output stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * <p>This constructor performs the reflective tests that are necessary to support {@link #position(long)}.
+ *
+ * @param os the output stream to wrap.
+ */
+ public ElfOutputBitStream(final OutputStream os) {
+ this(os, true);
+ }
+
+ /** Creates a new output bit stream wrapping a given output stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * @param os the output stream to wrap.
+ * @param testForPosition if false, the reflective test that is necessary to support {@link #position(long)}
+ * in case <code>os</code> does not support {@link RepositionableStream} will not be performed.
+ */
+ public ElfOutputBitStream(final OutputStream os, final boolean testForPosition) {
+ this(os, DEFAULT_BUFFER_SIZE);
+ }
+
+
+ /** Creates a new output bit stream wrapping a given output stream with a specified buffer size.
+ *
+ * <p>This constructor performs the reflective tests that are necessary to support {@link #position(long)}.
+ *
+ * @param os the output stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfOutputBitStream(final OutputStream os, final int bufSize) {
+ this(os, bufSize, true);
+ }
+
+ /** Creates a new output bit stream wrapping a given output stream with a specified buffer size.
+ *
+ * @param os the output stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ * @param testForPosition if false, the reflective test that is necessary to support {@link #position(long)}
+ * in case <code>os</code> does not support {@link RepositionableStream} will not be performed.
+ */
+ public ElfOutputBitStream(final OutputStream os, final int bufSize, final boolean testForPosition) {
+ this.os = os;
+ wrapping = false;
+ if (bufSize != 0) {
+ this.buffer = new byte[bufSize];
+ avail = bufSize;
+ }
+ free = 8;
+
+ if (os instanceof RepositionableStream) {
+ repositionableStream = (RepositionableStream)os;
+ fileChannel = null;
+ }
+ else if (testForPosition) {
+ FileChannel fc = null;
+ try {
+ fc = (FileChannel)(os.getClass().getMethod("getChannel")).invoke(os, new Object[] {});
+ }
+ catch(final IllegalAccessException e) {}
+ catch(final IllegalArgumentException e) {}
+ catch(final NoSuchMethodException e) {}
+ catch(final InvocationTargetException e) {}
+ catch(final ClassCastException e) {}
+ fileChannel = fc;
+ repositionableStream = null;
+ }
+ else {
+ repositionableStream = null;
+ fileChannel = null;
+ }
+ }
+
+ /** Creates a new output bit stream wrapping a given file output stream using a buffer of size {@link #DEFAULT_BUFFER_SIZE}.
+ *
+ * <p>This constructor invokes directly {@link FileOutputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param os the output stream to wrap.
+ */
+ public ElfOutputBitStream(final FileOutputStream os) {
+ this(os, DEFAULT_BUFFER_SIZE);
+ }
+
+ /** Creates a new output bit stream wrapping a given file output stream with a specified buffer size.
+ *
+ * <p>This constructor invokes directly {@link FileOutputStream#getChannel()} to support {@link #position(long)}.
+ *
+ * @param os the output stream to wrap.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfOutputBitStream(final FileOutputStream os, final int bufSize) {
+ this.os = os;
+ wrapping = false;
+ if (bufSize != 0) {
+ this.buffer = new byte[bufSize];
+ avail = bufSize;
+ }
+ free = 8;
+ repositionableStream = null;
+ fileChannel = os.getChannel();
+ }
+
+
+ /** Creates a new output bit stream wrapping a given byte array.
+ *
+ * @param a the byte array to wrap.
+ */
+ public ElfOutputBitStream(final byte[] a) {
+ os = null;
+ free = 8;
+ buffer = a;
+ avail = a.length;
+ wrapping = true;
+ fileChannel = null;
+ repositionableStream = null;
+ }
+
+ /** Creates a new output bit stream writing to file.
+ *
+ * @param name the name of the file.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfOutputBitStream(final String name, final int bufSize) throws FileNotFoundException {
+ this(new FileOutputStream(name), bufSize);
+ }
+
+ /** Creates a new output bit stream writing to a file.
+ *
+ * @param name the name of the file.
+ */
+ public ElfOutputBitStream(final String name) throws FileNotFoundException {
+ this(new FileOutputStream(name), DEFAULT_BUFFER_SIZE);
+ }
+
+
+ /** Creates a new output bit stream writing to file.
+ *
+ * @param file the file.
+ * @param bufSize the size in byte of the buffer; it may be 0, denoting no buffering.
+ */
+ public ElfOutputBitStream(final File file, final int bufSize) throws FileNotFoundException {
+ this(new FileOutputStream(file), bufSize);
+ }
+
+ /** Creates a new output bit stream writing to a file.
+ *
+ * @param file the file.
+ */
+ public ElfOutputBitStream(final File file) throws FileNotFoundException {
+ this(new FileOutputStream(file), DEFAULT_BUFFER_SIZE);
+ }
+
+
+ /** Flushes the bit stream.
+ *
+ * <P>This method will align the stream, write the bit buffer, empty the
+ * byte buffer and delegate to the {@link OutputStream#flush()} method of
+ * the underlying output stream.
+ *
+ * <P>This method is provided so that users of this class can easily wrap
+ * repositionable streams (for instance, file-based streams, which can be
+ * repositioned using the underlying {@link
+ * FileChannel}). <P> It is guaranteed that after calling
+ * this method the underlying stream can be repositioned, and that the next
+ * write to the underlying output stream will start with the content of the
+ * first write method called afterwards.
+ */
+
+ @Override
+ public void flush() {
+ try {
+ align();
+ if (os != null) {
+ if (buffer != null) {
+ os.write(buffer, 0, pos);
+ position += pos;
+ pos = 0;
+ avail = buffer.length;
+ }
+ os.flush();
+ }
+ } catch (Exception e) {
+ }
+
+ }
+
+
+ /** Closes the bit stream. All resources associated with the stream are released.
+ */
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ if (os != null && os != System.out && os != System.err) os.close();
+ buffer = null;
+ }
+
+ /** Returns the number of bits written to this bit stream.
+ *
+ * @return the number of bits written so far.
+ */
+ public long writtenBits() {
+ return writtenBits;
+ }
+
+ /** Sets the number of bits written to this bit stream.
+ *
+ * <P>This method is provided so that, for instance, the
+ * user can reset via <code>writtenBits(0)</code> the written-bits count
+ * after a {@link #flush()}.
+ *
+ * @param writtenBits the new value for the number of bits written so far.
+ */
+ public void writtenBits(final long writtenBits) {
+ this.writtenBits = writtenBits;
+ }
+
+ /** Writes a byte to the stream.
+ *
+ * <P>This method takes care of managing the buffering logic transparently.
+ *
+ * <P>However, this method does <em>not</em> update {@link #writtenBits}.
+ * The caller should increment {@link #writtenBits} by 8 at each call.
+ */
+
+ private void write(final int b) throws IOException {
+ if (avail-- == 0) {
+ if (buffer == null) {
+ os.write(b);
+ position++;
+ avail = 0;
+ return;
+ }
+ os.write(buffer);
+ position += buffer.length;
+ avail = buffer.length - 1;
+ pos = 0;
+ }
+
+ buffer[pos++] = (byte)b;
+ }
+
+
+ /** Writes bits in the bit buffer, possibly flushing it.
+ *
+ * You cannot write more than {@link #free} bits with this method. However,
+ * after having written {@link #free} bits the bit buffer will be empty. In
+ * particular, there should never be 0 free bits in the buffer.
+ *
+ * @param b the bits to write in the <strong>lower</strong> positions; the remaining positions must be zero.
+ * @param len the number of bits to write (0 is safe and causes no action).
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if one tries to write more bits than available in the buffer and debug is enabled.
+ */
+
+ private int writeInCurrent(final int b, final int len) throws IOException {
+
+ current |= (b & ((1 << len) - 1)) << (free -= len);
+ if (free == 0) {
+ write(current);
+ free = 8;
+ current = 0;
+ }
+
+ writtenBits += len;
+ return len;
+ }
+
+
+
+ /** Aligns the stream.
+ *
+ * After a call to this method, the stream is byte aligned. Zeroes
+ * are used to pad it if necessary.
+ *
+ * @return the number of padding bits.
+ */
+
+ public int align() throws IOException {
+ if (free != 8) return writeInCurrent(0, free);
+ else return 0;
+ }
+
+ /** Sets this stream bit position, if it is based on a {@link RepositionableStream} or on a {@link FileChannel}.
+ *
+ * <P>Given an underlying stream that implements {@link
+ * RepositionableStream} or that can provide a {@link
+ * FileChannel} via the <code>getChannel()</code> method,
+ * a call to this method has the same semantics of a {@link #flush()},
+ * followed by a call to {@link
+ * FileChannel#position(long) position(position / 8)} on
+ * the byte stream. Currently there is no clean, working way of supporting
+ * out-of-byte-boundary positioning.
+ *
+ * @param position the new position expressed as a bit offset; it must be byte-aligned.
+ * @throws IllegalArgumentException when trying to position outside of byte boundaries.
+ * @throws UnsupportedOperationException if the underlying byte stream does not implement
+ * {@link RepositionableStream} and if the channel it returns is not a {@link FileChannel}.
+ * @see FileChannel#position(long)
+ */
+
+ public void position(final long position) throws IOException {
+
+ if (position < 0) throw new IllegalArgumentException("Illegal position: " + position);
+ if ((position & 7) != 0) throw new IllegalArgumentException("Not a byte-aligned position: " + position);
+
+ if (wrapping) {
+ if ((position >>> 3) > buffer.length) throw new IllegalArgumentException("Illegal position: " + position);
+ flush();
+ free = 8;
+ pos = (int)(position >>> 3);
+ avail = buffer.length - pos;
+ }
+ else if (repositionableStream != null) {
+ flush();
+ if (position >>> 3 != this.position) repositionableStream.position(this.position = position >>> 3);
+ }
+ else if (fileChannel != null) {
+ flush();
+ if (position >>> 3 != this.position) fileChannel.position(this.position = position >>> 3);
+ }
+ else throw new UnsupportedOperationException("position() can only be called if the underlying byte stream implements the RepositionableStream interface or if the getChannel() method of the underlying byte stream exists and returns a FileChannel");
+ }
+
+ /** Writes a sequence of bits.
+ *
+ * Bits will be written in the natural way: the first bit is bit 7 of the
+ * first byte, the eightth bit is bit 0 of the first byte, the ninth bit is
+ * bit 7 of the second byte and so on.
+ *
+ * @param bits a vector containing the bits to be written.
+ * @param len a bit length.
+ * @return the number of bits written (<code>len</code>).
+ */
+ public long write(final byte[] bits, final long len) throws IOException {
+ return writeByteOffset(bits, 0, len);
+ }
+
+
+ /** Writes a sequence of bits, starting from a given offset.
+ *
+ * Bits will be written in the natural way: the first bit is bit 7 of the
+ * first byte, the eightth bit is bit 0 of the first byte, the ninth bit is
+ * bit 7 of the second byte and so on.
+ *
+ * @param bits a vector containing the bits to be written.
+ * @param offset a bit offset from which to start to write.
+ * @param len a bit length.
+ * @return the number of bits written (<code>len</code>).
+ */
+
+ public long write(final byte[] bits, final long offset, final long len) throws IOException {
+ final int initial = (int)(8 - (offset & 0x7));
+ if (initial == 8) return writeByteOffset(bits, (int)offset >>> 3, len);
+ if (len <= initial) return writeInt((0xFF & bits[(int)(offset >>> 3)]) >>> (initial - len), (int)len);
+ return writeInt(bits[(int)(offset >>> 3)], initial) + writeByteOffset(bits, (int)((offset >>> 3) + 1), len - initial);
+ }
+
+
+ /** Writes a sequence of bits, starting from a given byte offset.
+ *
+ * Bits will be written in the natural way: the first bit is bit 7 of the
+ * first byte, the eightth bit is bit 0 of the first byte, the ninth bit is
+ * bit 7 of the second byte and so on.
+ *
+ * <p>This method is used to support methods such as {@link #write(byte[], long, long)}.
+ *
+ * @param bits a vector containing the bits to be written.
+ * @param offset an offset, expressed in <strong>bytes</strong>.
+ * @param len a bit length.
+ * @return the number of bits written (<code>len</code>).
+ */
+
+ protected long writeByteOffset(final byte[] bits, final int offset, long len) throws IOException {
+
+ if (len == 0) return 0;
+ if (len <= free) return writeInCurrent(bits[offset] >>> 8 - len, (int)len);
+ else {
+ final int shift = free;
+ int i, j;
+
+ writeInCurrent(bits[offset] >>> 8 - shift, shift);
+
+ len -= shift;
+
+ j = offset;
+ i = (int)(len >> 3);
+ while(i-- != 0) {
+ write(bits[j] << shift | (bits[j + 1] & 0xFF) >>> 8 - shift);
+ writtenBits += 8;
+ j++;
+ }
+
+ final int queue = (int)(len & 7);
+ if (queue != 0) if (queue <= 8 - shift) writeInCurrent(bits[j] >>> 8 - shift - queue, queue);
+ else {
+ writeInCurrent(bits[j], 8 - shift);
+ writeInCurrent(bits[j + 1] >>> 16 - queue - shift, queue + shift - 8);
+ }
+
+ return len + shift;
+ }
+
+ }
+
+
+ /** Writes a bit.
+ *
+ * @param bit a bit.
+ * @return the number of bits written.
+ */
+
+ public int writeBit(final boolean bit) {
+ try {
+ return writeInCurrent(bit ? 1 : 0, 1);
+ } catch (IOException e) {
+ return -1;
+ }
+ }
+
+ /** Writes a bit.
+ *
+ * @param bit a bit.
+ * @return the number of bits written.
+ */
+
+ public int writeBit(final int bit) throws IOException {
+ if (bit < 0 || bit > 1) throw new IllegalArgumentException("The argument " + bit + " is not a bit.");
+ return writeInCurrent(bit, 1);
+ }
+
+ /** Writes a sequence of bits emitted by a boolean iterator.
+ *
+ * <P>If the iterator throws an exception, it is catched,
+ * and the return value is given by the number of bits written
+ * increased by one and with the sign changed.
+ *
+ * @param i a boolean iterator.
+ * @return if <code>i</code> did not throw a runtime exception,
+ * the number of bits written; otherwise, the number of bits written,
+ * plus one, with the sign changed.
+ */
+
+ public int write(final BooleanIterator i) throws IOException {
+ int count = 0;
+ boolean bit;
+ while(i.hasNext()) {
+ try {
+ bit = i.nextBoolean();
+ }
+ catch(final RuntimeException hide) {
+ return -count - 1;
+ }
+
+ writeBit(bit);
+ count++;
+ }
+ return count;
+ }
+
+
+ /** Writes a fixed number of bits from an integer.
+ *
+ * @param x an integer.
+ * @param len a bit length; this many lower bits of the first argument will be written
+ * (the most significant bit first).
+ * @return the number of bits written (<code>len</code>).
+ */
+
+ public int writeInt(int x, final int len) {
+
+ try {
+ if (len <= free) return writeInCurrent(x, len);
+
+ int i = len - free;
+ final int queue = i & 7;
+
+ if (free != 0) writeInCurrent(x >>> i, free);
+
+ // Dirty trick: since queue < 8, we pre-write the last bits in the bit buffer.
+ if (queue != 0) {
+ i -= queue;
+ writeInCurrent(x, queue);
+ x >>>= queue;
+ }
+
+ if (i == 32) write(x >>> 24);
+ if (i > 23) write(x >>> 16);
+ if (i > 15) write(x >>> 8);
+ if (i > 7) write(x);
+
+ writtenBits += i;
+
+ return len;
+
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ /** Writes a fixed number of bits from a long.
+ *
+ * @param x a long.
+ * @param len a bit length; this many lower bits of the first argument will be written
+ * (the most significant bit first).
+ * @return the number of bits written (<code>len</code>).
+ */
+
+ public int writeLong(long x, final int len) {
+ try {
+ if (len <= free) return writeInCurrent((int)x, len);
+
+ int i = len - free;
+ final int queue = i & 7;
+
+ if (free != 0) writeInCurrent((int)(x >>> i), free);
+
+ // Dirty trick: since queue < 8, we pre-write the last bits in the bit buffer.
+ if (queue != 0) {
+ i -= queue;
+ writeInCurrent((int)x, queue);
+ x >>>= queue;
+ }
+
+ if (i == 64) write((int)(x >>> 56));
+ if (i > 55) write((int)(x >>> 48));
+ if (i > 47) write((int)(x >>> 40));
+ if (i > 39) write((int)(x >>> 32));
+ if (i > 31) write((int)x >>> 24);
+ if (i > 23) write((int)x >>> 16);
+ if (i > 15) write((int)x >>> 8);
+ if (i > 7) write((int)x);
+
+ writtenBits += i;
+
+ return len;
+
+ } catch (Exception e) {
+ return -1;
+ }
+
+ }
+
+ /** Writes a natural number in unary coding.
+ *
+ * <p>The unary coding of a natural number <var>n</var> is given
+ * by 0<sup><var>n</var></sup>1.
+ *
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ */
+
+ public int writeUnary(int x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+
+ if (x < free) return writeInCurrent(1, x + 1);
+
+ final int shift = free;
+ x -= shift;
+
+ writtenBits += shift;
+ write(current);
+ free = 8;
+ current = 0;
+
+ int i = x >> 3;
+
+ writtenBits += (x & 0x7FFFFFF8);
+
+ while(i-- != 0) write(0);
+
+ writeInCurrent(1, (x & 7) + 1);
+
+ return x + shift + 1;
+ }
+
+ /** Writes a long natural number in unary coding.
+ *
+ * @param x a long natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeUnary(int)
+ */
+
+ public long writeLongUnary(long x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+
+ if (x < free) return writeInCurrent(1, (int)x + 1);
+
+ final int shift = free;
+ x -= shift;
+
+ writtenBits += shift;
+ write(current);
+ free = 8;
+ current = 0;
+
+ long i = x >> 3;
+
+ writtenBits += (x & 0x7FFFFFFFFFFFFFF8L);
+
+ while(i-- != 0) write(0);
+
+ writeInCurrent(1, (int)(x & 7) + 1);
+
+ return x + shift + 1;
+ }
+
+ /** Writes a natural number in γ coding.
+ *
+ * <P>The γ coding of a positive number of <var>k</var> bits is
+ * obtained writing <var>k</var>-1 in unary, followed by the lower
+ * <var>k</var>-1 bits of the number. The coding of a natural number is
+ * obtained by adding one and coding.
+ *
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ */
+
+ public int writeGamma(int x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(GAMMA[x], GAMMA[x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ return writeUnary(msb) + writeInt(x, msb);
+ }
+
+ /** Writes a given amount of natural numbers in γ coding.
+ *
+ * @param a an array at least <code>count</code> natural numbers.
+ * @param count the number of elements of <code>a</code> to be written.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeGamma(int)
+ */
+
+ public long writeGammas(final int[] a, final int count) throws IOException {
+ long l = 0;
+ for(int i = 0; i < count; i++) {
+ int x = a[i];
+ if (x < MAX_PRECOMPUTED) {
+ l += writeInt(GAMMA[x], GAMMA[x] >>> 26);
+ continue;
+ }
+
+ final int msb = Fast.mostSignificantBit(++x);
+ l += writeUnary(msb) + writeInt(x, msb);
+ }
+ return l;
+ }
+
+ /** Writes a long natural number in γ coding.
+ *
+ * @param x a long natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeGamma(int)
+ */
+
+ public int writeLongGamma(long x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(GAMMA[(int)x], GAMMA[(int)x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ return writeUnary(msb) + writeLong(x, msb);
+ }
+
+ /** Writes a natural number in shifted γ coding.
+ *
+ * The shifted γ coding of 0 is 1. The coding of a positive number
+ * of <var>k</var> bits is
+ * obtained writing <var>k</var> in unary, followed by the lower
+ * <var>k</var>-1 bits of the number (equivalently, by writing
+ * <var>k</var> zeroes followed by the number).
+ *
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ */
+
+ public int writeShiftedGamma(final int x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(SHIFTED_GAMMA[x], SHIFTED_GAMMA[x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(x);
+ return writeUnary(msb + 1) + (msb > 0 ? writeInt(x, msb) : 0);
+ }
+
+ /** Writes a long natural number in shifted γ coding.
+ *
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeShiftedGamma(int)
+ */
+
+ public int writeLongShiftedGamma(final long x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(SHIFTED_GAMMA[(int)x], SHIFTED_GAMMA[(int)x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(x);
+ return writeUnary(msb + 1) + (msb > 0 ? writeLong(x, msb) : 0);
+ }
+
+ /** Writes a given amount of natural numbers in shifted γ coding.
+ *
+ * @param a an array at least <code>count</code> natural numbers.
+ * @param count the number of elements of <code>a</code> to be written.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeShiftedGamma(int)
+ */
+
+ public long writeShiftedGammas(final int[] a, final int count) throws IOException {
+ long l = 0;
+ for(int i = 0; i < count; i++) {
+ final int x = a[i];
+ if (x < MAX_PRECOMPUTED) {
+ l += writeInt(SHIFTED_GAMMA[x], SHIFTED_GAMMA[x] >>> 26);
+ continue;
+ }
+
+ final int msb = Fast.mostSignificantBit(x);
+ l += writeUnary(msb + 1) + (msb > 0 ? writeInt(x, msb) : 0);
+ }
+ return l;
+ }
+
+ /** Writes a natural number in δ coding.
+ *
+ * The δ coding of a positive number of <var>k</var> bits is
+ * obtained writing <var>k</var>-1 in γ coding, followed by the
+ * lower <var>k</var>-1 bits of the number. The coding of a natural
+ * number is obtained by adding one and coding.
+ *
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ */
+
+ public int writeDelta(int x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(DELTA[x], DELTA[x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ return writeGamma(msb) + writeInt(x, msb);
+ }
+
+ /** Writes a long natural number in δ coding.
+ *
+ * @param x a long natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeDelta(int)
+ */
+
+ public int writeLongDelta(long x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (x < MAX_PRECOMPUTED) return writeInt(DELTA[(int)x], DELTA[(int)x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ return writeGamma(msb) + writeLong(x, msb);
+ }
+
+ /** Writes a given amount of natural numbers in δ coding.
+ *
+ * @param a an array at least <code>count</code> natural numbers.
+ * @param count the number of elements of <code>a</code> to be written.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeDelta(int)
+ */
+
+ public long writeDeltas(final int[] a, final int count) throws IOException {
+ long l = 0;
+ for(int i = 0; i < count; i++) {
+ int x = a[i];
+ if (x < MAX_PRECOMPUTED) {
+ l += writeInt(DELTA[x], DELTA[x] >>> 26);
+ continue;
+ }
+
+ final int msb = Fast.mostSignificantBit(++x);
+ l += writeGamma(msb) + writeInt(x, msb);
+ }
+ return l;
+ }
+
+ /** Writes a natural number in a limited range using a minimal binary coding.
+ *
+ * <p>A minimal binary code is an optimal code for the uniform distribution.
+ * This method uses an optimal code in which shorter words are assigned to
+ * smaller integers.
+ *
+ * @param x a natural number.
+ * @param b a strict upper bound for <code>x</code>.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive base.
+ */
+
+ public int writeMinimalBinary(final int x, final int b) throws IOException {
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+
+ return writeMinimalBinary(x, b, Fast.mostSignificantBit(b));
+ }
+
+ /** Writes a natural number in a limited range using a minimal binary coding.
+ *
+ * This method is faster than {@link #writeMinimalBinary(int,int)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param x a natural number.
+ * @param b a strict upper bound for <code>x</code>.
+ * @param log2b the floor of the base-2 logarithm of the bound.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive base.
+ * @see #writeMinimalBinary(int, int)
+ */
+
+ public int writeMinimalBinary(final int x, final int b, final int log2b) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+ if (x >= b) throw new IllegalArgumentException("The argument " + x + " exceeds the bound " + b);
+
+ // Numbers smaller than m are encoded in log2b bits.
+ final int m = (1 << log2b + 1) - b;
+
+ if (x < m) return writeInt(x, log2b);
+ else return writeInt(m + x, log2b + 1);
+ }
+
+
+ /** Writes a long natural number in a limited range using a minimal binary coding.
+ *
+ * @param x a natural number.
+ * @param b a strict upper bound for <code>x</code>.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive base.
+ * @see #writeMinimalBinary(int, int)
+ */
+
+ public int writeLongMinimalBinary(final long x, final long b) throws IOException {
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+
+ return writeLongMinimalBinary(x, b, Fast.mostSignificantBit(b));
+ }
+
+
+ /** Writes a long natural number in a limited range using a minimal binary coding.
+ *
+ * This method is faster than {@link #writeLongMinimalBinary(long,long)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param x a long natural number.
+ * @param b a strict upper bound for <code>x</code>.
+ * @param log2b the floor of the base-2 logarithm of the bound.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive base.
+ * @see #writeMinimalBinary(int, int)
+ */
+
+ public int writeLongMinimalBinary(final long x, final long b, final int log2b) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 1) throw new IllegalArgumentException("The bound " + b + " is not positive");
+ if (x >= b) throw new IllegalArgumentException("The argument " + x + " exceeds the bound " + b);
+
+ // Numbers smaller than m are encoded in log2b bits.
+ final long m = (1L << log2b + 1) - b;
+
+ if (x < m) return writeLong(x, log2b);
+ else return writeLong(m + x, log2b + 1);
+ }
+
+
+
+ /** Writes a natural number in Golomb coding.
+ *
+ * <p>Golomb coding with modulo <var>b</var> writes a natural number <var>x</var> as the quotient of
+ * the division of <var>x</var> and <var>b</var> in {@linkplain #writeUnary(int) unary},
+ * followed by the remainder in {@linkplain #writeMinimalBinary(int, int) minimal binary code}.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * the argument <code>x</code> may only be zero, and nothing will be written.
+ *
+ * @param x a natural number.
+ * @param b the modulus for the coding.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ */
+
+ public int writeGolomb(final int x, final int b) throws IOException {
+ return writeGolomb(x, b, Fast.mostSignificantBit(b));
+ }
+
+ /** Writes a natural number in Golomb coding.
+ *
+ * This method is faster than {@link #writeGolomb(int,int)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param x a natural number.
+ * @param b the modulus for the coding.
+ * @param log2b the floor of the base-2 logarithm of the coding modulus (it is irrelevant when <code>b</code> is zero).
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ * @see #writeGolomb(int, int)
+ */
+
+ public int writeGolomb(final int x, final int b, final int log2b) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) {
+ if (x != 0) throw new IllegalArgumentException("The modulus is 0, but the argument is " + x);
+ return 0;
+ }
+
+ final int l = writeUnary(x / b);
+
+ // The remainder to be encoded.
+ return l + writeMinimalBinary(x % b, b, log2b);
+ }
+
+
+
+ /** Writes a long natural number in Golomb coding.
+ *
+ * @param x a long natural number.
+ * @param b the modulus for the coding.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ * @see #writeGolomb(int, int)
+ */
+
+ public long writeLongGolomb(final long x, final long b) throws IOException {
+ return writeLongGolomb(x, b, Fast.mostSignificantBit(b));
+ }
+
+ /** Writes a long natural number in Golomb coding.
+ *
+ * This method is faster than {@link #writeLongGolomb(long,long)} because it does not
+ * have to compute <code>log2b</code>.
+ *
+ * @param x a long natural number.
+ * @param b the modulus for the coding.
+ * @param log2b the floor of the base-2 logarithm of the coding modulus (it is irrelevant when <code>b</code> is zero).
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ * @see #writeGolomb(int, int)
+ */
+
+ public long writeLongGolomb(final long x, final long b, final int log2b) throws IOException {
+
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) {
+ if (x != 0) throw new IllegalArgumentException("The modulus is 0, but the argument is " + x);
+ return 0;
+ }
+
+ final long l = writeLongUnary(x / b);
+
+ // The remainder to be encoded.
+ return l + writeLongMinimalBinary(x % b, b, log2b);
+ }
+
+
+ /** Writes a natural number in skewed Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * the argument <code>x</code> may only be zero, and nothing will be written.
+ *
+ * @param x a natural number.
+ * @param b the modulus for the coding.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ */
+
+ public int writeSkewedGolomb(final int x, final int b) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) {
+ if (x != 0) throw new IllegalArgumentException("The modulus is 0, but the argument is " + x);
+ return 0;
+ }
+
+ final int i = Fast.mostSignificantBit(x / b + 1);
+ final int l = writeUnary(i);
+ final int M = ((1 << i + 1) - 1) * b;
+ final int m = (M / (2 * b)) * b;
+
+ return l + writeMinimalBinary(x - m, M - m);
+ }
+
+ /** Writes a long natural number in skewed Golomb coding.
+ *
+ * <P>This method implements also the case in which <code>b</code> is 0: in this case,
+ * the argument <code>x</code> may only be zero, and nothing will be written.
+ *
+ * @param x a long natural number.
+ * @param b the modulus for the coding.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a negative modulus.
+ * @see #writeSkewedGolomb(int, int)
+ */
+
+ public long writeLongSkewedGolomb(final long x, final long b) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (b < 0) throw new IllegalArgumentException("The modulus " + b + " is negative");
+ if (b == 0) {
+ if (x != 0) throw new IllegalArgumentException("The modulus is 0, but the argument is " + x);
+ return 0;
+ }
+
+ final long i = Fast.mostSignificantBit(x / b + 1);
+ final long l = writeLongUnary(i);
+ final long M = ((1L << i + 1) - 1) * b;
+ final long m = (M / (2 * b)) * b;
+
+ return l + writeLongMinimalBinary(x - m, M - m);
+ }
+
+
+ /** Writes a natural number in ζ coding.
+ *
+ * <P>ζ coding (with modulo <var>k</var>) records positive numbers in
+ * the intervals
+ * [1,2<sup><var>k</var></sup>-1],[2<sup><var>k</var></sup>,2<sup><var>k</var>+1</sup>-1],…,[2<sup><var>hk</var></sup>,2<sup>(<var>h</var>+1)<var>k</var></sup>-1]
+ * by coding <var>h</var> in unary, followed by a minimal binary coding of
+ * the offset in the interval. The coding of a natural number is obtained
+ * by adding one and coding.
+ *
+ * <P>ζ codes were defined by
+ * Paolo Boldi and Sebastiano Vigna in
+ * “<a href="http://vigna.di.unimi.it/papers.php#BoVCWWW">Codes for the World−Wide Web</a>”,
+ * <i>Internet Math.</i>, 2(4):405-427, 2005. The paper contains also a detailed analysis.
+ *
+ * @param x a natural number.
+ * @param k the shrinking factor.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive shrinking factor.
+ */
+
+ public int writeZeta(int x, final int k) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (k < 1) throw new IllegalArgumentException("The shrinking factor " + k + " is not positive");
+ if (k == 3 && x < MAX_PRECOMPUTED) return writeInt(ZETA_3[x], ZETA_3[x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ final int h = msb / k;
+ final int l = writeUnary(h);
+ final int left = 1 << h * k;
+ return l + (x - left < left
+ ? writeInt(x - left, h * k + k - 1)
+ : writeInt(x, h * k + k));
+ }
+
+ /** Writes a long natural number in ζ coding.
+ *
+ * @param x a long natural number.
+ * @param k the shrinking factor.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number or use a nonpositive shrinking factor.
+ * @see #writeZeta(int, int)
+ */
+
+ public int writeLongZeta(long x, final int k) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+ if (k < 1) throw new IllegalArgumentException("The shrinking factor " + k + " is not positive");
+ if (k == 3 && x < MAX_PRECOMPUTED) return writeInt(ZETA_3[(int)x], ZETA_3[(int)x] >>> 26);
+
+ final int msb = Fast.mostSignificantBit(++x);
+ final int h = msb / k;
+ final int l = writeUnary(h);
+ final long left = 1L << h * k;
+ return l + (x - left < left
+ ? writeLong(x - left, h * k + k - 1)
+ : writeLong(x, h * k + k));
+ }
+
+
+ /** Writes a natural number in variable-length nibble coding.
+ *
+ * <P>Variable-length nibble coding records a natural number by padding its binary
+ * representation to the left using zeroes, until its length is a multiple of three.
+ * Then, the resulting string is
+ * broken in blocks of 3 bits, and each block is prefixed with a bit, which is
+ * zero for all blocks except for the last one.
+ * @param x a natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ */
+
+ public int writeNibble(final int x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+
+ if (x == 0) return writeInt(8, 4);
+ final int msb = Fast.mostSignificantBit(x);
+ int h = msb / 3;
+ do {
+ writeBit(h == 0);
+ writeInt(x >> h * 3 , 3);
+ } while(h-- != 0);
+ return ((msb / 3) + 1) << 2;
+ }
+
+ /** Writes a long natural number in variable-length nibble coding.
+ *
+ * @param x a long natural number.
+ * @return the number of bits written.
+ * @throws IllegalArgumentException if you try to write a negative number.
+ * @see #writeNibble(int)
+ */
+
+ public int writeLongNibble(final long x) throws IOException {
+ if (x < 0) throw new IllegalArgumentException("The argument " + x + " is negative");
+
+ if (x == 0) return writeInt(8, 4);
+ final int msb = Fast.mostSignificantBit(x);
+ int h = msb / 3;
+ do {
+ writeBit(h == 0);
+ writeInt((int)(x >> h * 3) , 3);
+ } while(h-- != 0);
+ return ((msb / 3) + 1) << 2;
+ }
+
+ /** Copies a given number of bits from a given input bit stream into this output bit stream.
+ *
+ * @param ibs an input bit stream.
+ * @param length the number of bits to copy.
+ * @throws EOFException if there are not enough bits to copy.
+ */
+ public void copyFrom(final ElfInputBitStream ibs, long length) throws IOException {
+ final byte[] buffer = new byte[64 * 1024];
+ while(length > 0) {
+ final int toRead = (int)Math.min(length, buffer.length * Byte.SIZE);
+ ibs.read(buffer, toRead);
+ write(buffer, 0, toRead);
+ length -= toRead;
+ }
+ }
+
+ public byte[] getBuffer() {
+ return buffer;
+ }
+
+ /** Bytes written to the wrapped array after {@link #flush()}. */
+ public int getByteLength() {
+ return pos;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/AbstractElfCompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/AbstractElfCompressor.java
new file mode 100644
index 0000000..10a02f0
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/AbstractElfCompressor.java
@@ -0,0 +1,54 @@
+package org.apache.iotdb.tsfile.encoding.elf.compressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.utils.Elf64Utils;
+
+public abstract class AbstractElfCompressor implements ICompressor {
+ private int size = 0;
+
+ private int lastBetaStar = Integer.MAX_VALUE;
+
+ public void addValue(double v) {
+ long vLong = Double.doubleToRawLongBits(v);
+ long vPrimeLong;
+
+ if (v == 0.0 || Double.isInfinite(v)) {
+ size += writeInt(2, 2); // case 10
+ vPrimeLong = vLong;
+ } else if (Double.isNaN(v)) {
+ size += writeInt(2, 2); // case 10
+ vPrimeLong = 0x7ff8000000000000L;
+ } else {
+ // C1: v is a normal or subnormal
+ int[] alphaAndBetaStar = Elf64Utils.getAlphaAndBetaStar(v, lastBetaStar);
+ int e = ((int) (vLong >> 52)) & 0x7ff;
+ int gAlpha = Elf64Utils.getFAlpha(alphaAndBetaStar[0]) + e - 1023;
+ int eraseBits = 52 - gAlpha;
+ long mask = 0xffffffffffffffffL << eraseBits;
+ long delta = (~mask) & vLong;
+ if (delta != 0 && eraseBits > 4) { // C2
+ if(alphaAndBetaStar[1] == lastBetaStar) {
+ size += writeBit(false); // case 0
+ } else {
+ size += writeInt(alphaAndBetaStar[1] | 0x30, 6); // case 11, 2 + 4 = 6
+ lastBetaStar = alphaAndBetaStar[1];
+ }
+ vPrimeLong = mask & vLong;
+ } else {
+ size += writeInt(2, 2); // case 10
+ vPrimeLong = vLong;
+ }
+ }
+ size += xorCompress(vPrimeLong);
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ protected abstract int writeInt(int n, int len);
+
+ protected abstract int writeBit(boolean bit);
+
+ protected abstract int xorCompress(long vPrimeLong);
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ElfCompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ElfCompressor.java
new file mode 100644
index 0000000..0ddd039
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ElfCompressor.java
@@ -0,0 +1,42 @@
+package org.apache.iotdb.tsfile.encoding.elf.compressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.chimp.ElfOutputBitStream;
+import org.apache.iotdb.tsfile.encoding.elf.xorcompressor.ElfXORCompressor;
+
+public class ElfCompressor extends AbstractElfCompressor {
+ private final ElfXORCompressor xorCompressor;
+
+ public ElfCompressor() {
+ xorCompressor = new ElfXORCompressor();
+ }
+
+ public ElfCompressor(int xorBufferBytes) {
+ xorCompressor = new ElfXORCompressor(xorBufferBytes);
+ }
+
+ @Override protected int writeInt(int n, int len) {
+ ElfOutputBitStream os = xorCompressor.getOutputStream();
+ os.writeInt(n, len);
+ return len;
+ }
+
+ @Override protected int writeBit(boolean bit) {
+ ElfOutputBitStream os = xorCompressor.getOutputStream();
+ os.writeBit(bit);
+ return 1;
+ }
+
+ @Override protected int xorCompress(long vPrimeLong) {
+ return xorCompressor.addValue(vPrimeLong);
+ }
+
+ @Override public byte[] getBytes() {
+ return xorCompressor.getOut();
+ }
+
+ @Override public void close() {
+ // we write one more bit here, for marking an end of the stream.
+ writeInt(2,2); // case 10
+ xorCompressor.close();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ICompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ICompressor.java
new file mode 100644
index 0000000..f6c15f8
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/compressor/ICompressor.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.elf.compressor;
+
+public interface ICompressor {
+ void addValue(double v);
+
+ int getSize();
+
+ byte[] getBytes();
+
+ void close();
+
+ default String getKey() {
+ return getClass().getSimpleName();
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/AbstractElfDecompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/AbstractElfDecompressor.java
new file mode 100644
index 0000000..a1b5f5f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/AbstractElfDecompressor.java
@@ -0,0 +1,54 @@
+package org.apache.iotdb.tsfile.encoding.elf.decompressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.utils.Elf64Utils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class AbstractElfDecompressor implements IDecompressor {
+ private int lastBetaStar = Integer.MAX_VALUE;
+
+ public List<Double> decompress() {
+ List<Double> values = new ArrayList<>(1024);
+ Double value;
+ while ((value = nextValue()) != null) {
+ values.add(value);
+ }
+ return values;
+ }
+
+ private Double nextValue() {
+ Double v;
+
+ if(readInt(1) == 0) {
+ v = recoverVByBetaStar(); // case 0
+ } else if (readInt(1) == 0) {
+ v = xorDecompress(); // case 10
+ } else {
+ lastBetaStar = readInt(4); // case 11
+ v = recoverVByBetaStar();
+ }
+ return v;
+ }
+
+ private Double recoverVByBetaStar() {
+ double v;
+ Double vPrime = xorDecompress();
+ int sp = Elf64Utils.getSP(Math.abs(vPrime));
+ if (lastBetaStar == 0) {
+ v = Elf64Utils.get10iN(-sp - 1);
+ if (vPrime < 0) {
+ v = -v;
+ }
+ } else {
+ int alpha = lastBetaStar - sp - 1;
+ v = Elf64Utils.roundUp(vPrime, alpha);
+ }
+ return v;
+ }
+
+ protected abstract Double xorDecompress();
+
+ protected abstract int readInt(int len);
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/ElfDecompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/ElfDecompressor.java
new file mode 100644
index 0000000..25fdbd5
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/ElfDecompressor.java
@@ -0,0 +1,27 @@
+package org.apache.iotdb.tsfile.encoding.elf.decompressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.chimp.ElfInputBitStream;
+import org.apache.iotdb.tsfile.encoding.elf.xordecompressor.ElfXORDecompressor;
+
+import java.io.IOException;
+
+public class ElfDecompressor extends AbstractElfDecompressor {
+ private final ElfXORDecompressor xorDecompressor;
+
+ public ElfDecompressor(byte[] bytes) {
+ xorDecompressor = new ElfXORDecompressor(bytes);
+ }
+
+ @Override protected Double xorDecompress() {
+ return xorDecompressor.readValue();
+ }
+
+ @Override protected int readInt(int len) {
+ ElfInputBitStream in = xorDecompressor.getInputStream();
+ try {
+ return in.readInt(len);
+ } catch (IOException e) {
+ throw new RuntimeException("IO error: " + e.getMessage());
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/IDecompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/IDecompressor.java
new file mode 100644
index 0000000..908195f
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/decompressor/IDecompressor.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.encoding.elf.decompressor;
+
+import java.util.List;
+
+public interface IDecompressor {
+ List<Double> decompress();
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/utils/Elf64Utils.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/utils/Elf64Utils.java
new file mode 100644
index 0000000..a2dedc7
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/utils/Elf64Utils.java
@@ -0,0 +1,163 @@
+package org.apache.iotdb.tsfile.encoding.elf.utils;
+
+public class Elf64Utils {
+ // αlog_2(10) for look-up
+ private final static int[] f =
+ new int[] {0, 4, 7, 10, 14, 17, 20, 24, 27, 30, 34, 37, 40, 44, 47, 50, 54, 57,
+ 60, 64, 67};
+
+ private final static double[] map10iP =
+ new double[] {1.0, 1.0E1, 1.0E2, 1.0E3, 1.0E4, 1.0E5, 1.0E6, 1.0E7,
+ 1.0E8, 1.0E9, 1.0E10, 1.0E11, 1.0E12, 1.0E13, 1.0E14,
+ 1.0E15, 1.0E16, 1.0E17, 1.0E18, 1.0E19, 1.0E20};
+
+ private final static double[] map10iN =
+ new double[] {1.0, 1.0E-1, 1.0E-2, 1.0E-3, 1.0E-4, 1.0E-5, 1.0E-6, 1.0E-7,
+ 1.0E-8, 1.0E-9, 1.0E-10, 1.0E-11, 1.0E-12, 1.0E-13, 1.0E-14,
+ 1.0E-15, 1.0E-16, 1.0E-17, 1.0E-18, 1.0E-19, 1.0E-20};
+
+ private final static long[] mapSPGreater1 =
+ new long[] {1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000};
+
+ private final static double[] mapSPLess1 =
+ new double[] {1, 0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001, 0.0000001, 0.00000001,
+ 0.000000001, 0.0000000001};
+
+ private final static double LOG_2_10 = Math.log(10) / Math.log(2);
+
+ public static int getFAlpha(int alpha) {
+ if (alpha < 0) {
+ throw new IllegalArgumentException("The argument should be greater than 0");
+ }
+ if (alpha >= f.length) {
+ return (int) Math.ceil(alpha * LOG_2_10);
+ } else {
+ return f[alpha];
+ }
+ }
+
+ public static int[] getAlphaAndBetaStar(double v, int lastBetaStar) {
+ if (v < 0) {
+ v = -v;
+ }
+ int[] alphaAndBetaStar = new int[2];
+ int[] spAnd10iNFlag = getSPAnd10iNFlag(v);
+ int beta = getSignificantCount(v, spAnd10iNFlag[0], lastBetaStar);
+ alphaAndBetaStar[0] = beta - spAnd10iNFlag[0] - 1;
+ alphaAndBetaStar[1] = spAnd10iNFlag[1] == 1 ? 0 : beta;
+ return alphaAndBetaStar;
+ }
+
+ public static double roundUp(double v, int alpha) {
+ double scale = get10iP(alpha);
+ if (v < 0) {
+ return Math.floor(v * scale) / scale;
+ } else {
+ return Math.ceil(v * scale) / scale;
+ }
+ }
+
+ private static int getSignificantCount(double v, int sp, int lastBetaStar) {
+ int i;
+ if(lastBetaStar != Integer.MAX_VALUE && lastBetaStar != 0) {
+ i = Math.max(lastBetaStar - sp - 1, 1);
+ } else if (lastBetaStar == Integer.MAX_VALUE) {
+ i = 17 - sp - 1;
+ } else if (sp >= 0) {
+ i = 1;
+ } else {
+ i = -sp;
+ }
+
+ double temp = v * get10iP(i);
+ long tempLong = (long) temp;
+ while (tempLong != temp) {
+ i++;
+ temp = v * get10iP(i);
+ tempLong = (long) temp;
+ }
+
+ // There are some bugs for those with high significand, i.e., 0.23911204406033099
+ // So we should further check
+ if (temp / get10iP(i) != v) {
+ return 17;
+ } else {
+ while (i > 0 && tempLong % 10 == 0) {
+ i--;
+ tempLong = tempLong / 10;
+ }
+ return sp + i + 1;
+ }
+ }
+
+ private static double get10iP(int i) {
+ if (i < 0) {
+ throw new IllegalArgumentException("The argument should be greater than 0");
+ }
+ if (i >= map10iP.length) {
+ return Double.parseDouble("1.0E" + i);
+ } else {
+ return map10iP[i];
+ }
+ }
+
+ public static double get10iN(int i) {
+ if (i < 0) {
+ throw new IllegalArgumentException("The argument should be greater than 0");
+ }
+ if (i >= map10iN.length) {
+ return Double.parseDouble("1.0E-" + i);
+ } else {
+ return map10iN[i];
+ }
+ }
+
+ public static int getSP(double v) {
+ if (v >= 1) {
+ int i = 0;
+ while (i < mapSPGreater1.length - 1) {
+ if (v < mapSPGreater1[i + 1]) {
+ return i;
+ }
+ i++;
+ }
+ } else {
+ int i = 1;
+ while (i < mapSPLess1.length) {
+ if (v >= mapSPLess1[i]) {
+ return -i;
+ }
+ i++;
+ }
+ }
+ return (int) Math.floor(Math.log10(v));
+ }
+
+ private static int[] getSPAnd10iNFlag(double v) {
+ int[] spAnd10iNFlag = new int[2];
+ if (v >= 1) {
+ int i = 0;
+ while (i < mapSPGreater1.length - 1) {
+ if (v < mapSPGreater1[i + 1]) {
+ spAnd10iNFlag[0] = i;
+ return spAnd10iNFlag;
+ }
+ i++;
+ }
+ } else {
+ int i = 1;
+ while (i < mapSPLess1.length) {
+ if (v >= mapSPLess1[i]) {
+ spAnd10iNFlag[0] = -i;
+ spAnd10iNFlag[1] = v == mapSPLess1[i] ? 1 : 0;
+ return spAnd10iNFlag;
+ }
+ i++;
+ }
+ }
+ double log10v = Math.log10(v);
+ spAnd10iNFlag[0] = (int) Math.floor(log10v);
+ spAnd10iNFlag[1] = log10v == (long)log10v ? 1 : 0;
+ return spAnd10iNFlag;
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xorcompressor/ElfXORCompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xorcompressor/ElfXORCompressor.java
new file mode 100644
index 0000000..b5cee69
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xorcompressor/ElfXORCompressor.java
@@ -0,0 +1,165 @@
+package org.apache.iotdb.tsfile.encoding.elf.xorcompressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.chimp.ElfOutputBitStream;
+
+public class ElfXORCompressor {
+ private int storedLeadingZeros = Integer.MAX_VALUE;
+
+ private int storedTrailingZeros = Integer.MAX_VALUE;
+ private long storedVal = 0;
+ private boolean first = true;
+ private int size;
+ private final static long END_SIGN = Double.doubleToLongBits(Double.NaN);
+
+ public final static short[] leadingRepresentation = {0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 1, 1, 1, 2, 2, 2, 2,
+ 3, 3, 4, 4, 5, 5, 6, 6,
+ 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7,
+ 7, 7, 7, 7, 7, 7, 7, 7
+ };
+
+ public final static short[] leadingRound = {0, 0, 0, 0, 0, 0, 0, 0,
+ 8, 8, 8, 8, 12, 12, 12, 12,
+ 16, 16, 18, 18, 20, 20, 22, 22,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24,
+ 24, 24, 24, 24, 24, 24, 24, 24
+ };
+ // public final static short FIRST_DELTA_BITS = 27;
+
+ private final ElfOutputBitStream out;
+
+ public ElfXORCompressor() {
+ this(10000);
+ }
+
+ public ElfXORCompressor(int bufferBytes) {
+ out = new ElfOutputBitStream(new byte[Math.max(bufferBytes, 4096)]);
+ size = 0;
+ }
+
+ public ElfOutputBitStream getOutputStream() {
+ return this.out;
+ }
+
+ /**
+ * Adds a new long value to the series. Note, values must be inserted in order.
+ *
+ * @param value next floating point value in the series
+ */
+ public int addValue(long value) {
+ if (first) {
+ return writeFirst(value);
+ } else {
+ return compressValue(value);
+ }
+ }
+
+ /**
+ * Adds a new double value to the series. Note, values must be inserted in order.
+ *
+ * @param value next floating point value in the series
+ */
+ public int addValue(double value) {
+ if (first) {
+ return writeFirst(Double.doubleToRawLongBits(value));
+ } else {
+ return compressValue(Double.doubleToRawLongBits(value));
+ }
+ }
+
+ private int writeFirst(long value) {
+ first = false;
+ storedVal = value;
+ int trailingZeros = Long.numberOfTrailingZeros(value);
+ out.writeInt(trailingZeros, 7);
+ if (trailingZeros < 64) {
+ out.writeLong(storedVal >>> (trailingZeros + 1), 63 - trailingZeros);
+ size += 70 - trailingZeros;
+ return 70 - trailingZeros;
+ } else {
+ size += 7;
+ return 7;
+ }
+ }
+
+ /**
+ * Closes the block and writes the remaining stuff to the BitOutput.
+ */
+ public void close() {
+ addValue(END_SIGN);
+ out.writeBit(false);
+ out.flush();
+ }
+
+ private int compressValue(long value) {
+ int thisSize = 0;
+ long xor = storedVal ^ value;
+
+ if (xor == 0) {
+ // case 01
+ out.writeInt(1, 2);
+
+ size += 2;
+ thisSize += 2;
+ } else {
+ int leadingZeros = leadingRound[Long.numberOfLeadingZeros(xor)];
+ int trailingZeros = Long.numberOfTrailingZeros(xor);
+
+ if (leadingZeros == storedLeadingZeros && trailingZeros >= storedTrailingZeros) {
+ // case 00
+ int centerBits = 64 - storedLeadingZeros - storedTrailingZeros;
+ int len = 2 + centerBits;
+ if(len > 64) {
+ out.writeInt(0, 2);
+ out.writeLong(xor >>> storedTrailingZeros, centerBits);
+ } else {
+ out.writeLong(xor >>> storedTrailingZeros, len);
+ }
+
+ size += len;
+ thisSize += len;
+ } else {
+ storedLeadingZeros = leadingZeros;
+ storedTrailingZeros = trailingZeros;
+ int centerBits = 64 - storedLeadingZeros - storedTrailingZeros;
+
+ if (centerBits <= 16) {
+ // case 10
+ out.writeInt((((0x2 << 3) | leadingRepresentation[storedLeadingZeros]) << 4) | (centerBits & 0xf), 9);
+ out.writeLong(xor >>> (storedTrailingZeros + 1), centerBits - 1);
+
+ size += 8 + centerBits;
+ thisSize += 8 + centerBits;
+ } else {
+ // case 11
+ out.writeInt((((0x3 << 3) | leadingRepresentation[storedLeadingZeros]) << 6) | (centerBits & 0x3f), 11);
+ out.writeLong(xor >>> (storedTrailingZeros + 1), centerBits - 1);
+
+ size += 10 + centerBits;
+ thisSize += 10 + centerBits;
+ }
+ }
+
+ storedVal = value;
+ }
+
+ return thisSize;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public byte[] getOut() {
+ out.flush();
+ byte[] buf = out.getBuffer();
+ int n = out.getByteLength();
+ return java.util.Arrays.copyOf(buf, n);
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xordecompressor/ElfXORDecompressor.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xordecompressor/ElfXORDecompressor.java
new file mode 100644
index 0000000..fedd9df
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/elf/xordecompressor/ElfXORDecompressor.java
@@ -0,0 +1,129 @@
+package org.apache.iotdb.tsfile.encoding.elf.xordecompressor;
+
+import org.apache.iotdb.tsfile.encoding.elf.chimp.ElfInputBitStream;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ElfXORDecompressor {
+ private long storedVal = 0;
+ private int storedLeadingZeros = Integer.MAX_VALUE;
+ private int storedTrailingZeros = Integer.MAX_VALUE;
+ private boolean first = true;
+ private boolean endOfStream = false;
+
+ private final ElfInputBitStream in;
+
+ private final static long END_SIGN = Double.doubleToLongBits(Double.NaN);
+
+ private final static short[] leadingRepresentation = {0, 8, 12, 16, 18, 20, 22, 24};
+
+ public ElfXORDecompressor(byte[] bs) {
+ in = new ElfInputBitStream(bs);
+ }
+
+ public List<Double> getValues() {
+ List<Double> list = new ArrayList<>(1024);
+ Double value = readValue();
+ while (value != null) {
+ list.add(value);
+ value = readValue();
+ }
+ return list;
+ }
+
+ public ElfInputBitStream getInputStream() {
+ return in;
+ }
+
+ /**
+ * Returns the next pair in the time series, if available.
+ *
+ * @return Pair if there's next value, null if series is done.
+ */
+ public Double readValue() {
+ try {
+ next();
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ if (endOfStream) {
+ return null;
+ }
+ return Double.longBitsToDouble(storedVal);
+ }
+
+ private void next() throws IOException {
+ if (first) {
+ first = false;
+ int trailingZeros = in.readInt(7);
+ if (trailingZeros < 64) {
+ storedVal = ((in.readLong(63 - trailingZeros) << 1) + 1) << trailingZeros;
+ } else {
+ storedVal = 0;
+ }
+ if (storedVal == END_SIGN) {
+ endOfStream = true;
+ }
+ } else {
+ nextValue();
+ }
+ }
+
+ private void nextValue() throws IOException {
+ long value;
+ int centerBits, leadAndCenter;
+ int flag = in.readInt(2);
+ switch (flag) {
+ case 3:
+ // case 11
+ leadAndCenter = in.readInt(9);
+ storedLeadingZeros = leadingRepresentation[leadAndCenter >>> 6];
+ centerBits = leadAndCenter & 0x3f;
+ if(centerBits == 0) {
+ centerBits = 64;
+ }
+ storedTrailingZeros = 64 - storedLeadingZeros - centerBits;
+ value = ((in.readLong(centerBits - 1) << 1) + 1) << storedTrailingZeros;
+ value = storedVal ^ value;
+ if (value == END_SIGN) {
+ endOfStream = true;
+ } else {
+ storedVal = value;
+ }
+ break;
+ case 2:
+ // case 10
+ leadAndCenter = in.readInt(7);
+ storedLeadingZeros = leadingRepresentation[leadAndCenter >>> 4];
+ centerBits = leadAndCenter & 0xf;
+ if(centerBits == 0) {
+ centerBits = 16;
+ }
+ storedTrailingZeros = 64 - storedLeadingZeros - centerBits;
+ value = ((in.readLong(centerBits - 1) << 1) + 1) << storedTrailingZeros;
+ value = storedVal ^ value;
+ if (value == END_SIGN) {
+ endOfStream = true;
+ } else {
+ storedVal = value;
+ }
+ break;
+ case 1:
+ // case 01, we do nothing, the same value as before
+ break;
+ default:
+ // case 00
+ centerBits = 64 - storedLeadingZeros - storedTrailingZeros;
+ value = in.readLong(centerBits) << storedTrailingZeros;
+ value = storedVal ^ value;
+ if (value == END_SIGN) {
+ endOfStream = true;
+ } else {
+ storedVal = value;
+ }
+ break;
+ }
+ }
+}
