Icde25 revision submission (#13974)
Co-authored-by: gzh0806 <gzh23@mails.tsinghua.edu.cn>
diff --git a/iotdb-core/tsfile/pom.xml b/iotdb-core/tsfile/pom.xml
index 0a09da5..39f45ce 100644
--- a/iotdb-core/tsfile/pom.xml
+++ b/iotdb-core/tsfile/pom.xml
@@ -103,6 +103,14 @@
<runOrder>random</runOrder>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>16</source>
+ <target>16</target>
+ </configuration>
+ </plugin>
</plugins>
</build>
<dependencies>
@@ -181,6 +189,16 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.21</version> <!-- 请检查是否有更新的版本 -->
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ <dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>24.0.1</version>
@@ -196,6 +214,11 @@
<artifactId>JavaFastPFOR</artifactId>
<version>[0.1,)</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<profiles>
<profile>
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
index e3bc4b9..db3877a 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/EncodeTest.java
@@ -27,9 +27,9 @@
public static void main(@NotNull String[] args) throws IOException {
- //String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/sota_ratio/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sota_ratio/";
String input_parent_dir = parent_dir + "trans_data/";
ArrayList<String> input_path_list = new ArrayList<>();
ArrayList<String> output_path_list = new ArrayList<>();
@@ -82,7 +82,7 @@
String Output = output_path_list.get(file_i);
// speed
- int repeatTime = 100; // set repeat time
+ int repeatTime = 10; // set repeat time
String dataTypeName = "int"; // set dataType
File file = new File(inputPath);
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupL.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupL.java
deleted file mode 100644
index 3141984..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupL.java
+++ /dev/null
@@ -1,114 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static java.lang.Math.pow;
-
-public class GroupL {
- public int[] number;
- public int count;
- public int[] count_array;
- public int range;
- public long[] sorted_value_list;
- public int unique_number;
- public int mask;
- public int left_shift;
-
- public int getCount(long long1) {
- return ((int) (long1 & this.mask));
- }
- public int getUniqueValue(long long1) {
- return ((int) ((long1) >> this.left_shift));
- }
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
- public GroupL(int[] number, int count, int i) {
- this.number = number;
- this.count = count;
- this.range = (int) pow(2,i-1);
- this.count_array = new int[range];
- }
-
- public int[] getNumber() {
- return number;
- }
-
-
- public int getCount() {
- return count;
- }
-
- public void addNumber(int number) {
- this.number[this.count] = number;
- this.count++;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public void setCount_array(){
- double par = this.range/(count*Math.log(this.count));
- int k1_start = this.range;
- this.left_shift = getBitWith(this.count);
- this.mask = (1 << left_shift) - 1;
- if (par > 3) {
- int[] value_list = new int[this.count];
- for (int i = 0; i < this.count; i++) {
- int value = this.number[i];
- count_array[value - k1_start]++;
- if (count_array[value - k1_start] == 1) {
- value_list[unique_number] = value;
- unique_number++;
- }
- }
- sorted_value_list = new long[unique_number];
- for (int i = 0; i < unique_number; i++) {
- int value = value_list[i];
- sorted_value_list[i] = (((long) (value - k1_start)) << left_shift) + count_array[value - k1_start];
- }
- Arrays.sort(sorted_value_list);
-
- int cdf_count = 0;
- for (int i = 0; i < unique_number; i++) {
- cdf_count += getCount(sorted_value_list[i]);
- sorted_value_list[i] = (((long) getUniqueValue(sorted_value_list[i])) << left_shift) + cdf_count;//new_value_list[i]
- }
- }else{
- int[] hash = new int[this.range];
- for (int i = 0; i < this.count; i++) {
- int value = this.number[i];
- if (hash[value - k1_start] == 0){
- this.unique_number++;
- }
- hash[value - k1_start]++;
- }
- sorted_value_list = new long[unique_number];
- int ccount = 0;
- int index = 0;
- for(int i=0;i<this.range;i++){
- if (hash[i] >0) {
- ccount += hash[i];
- sorted_value_list[index] = (((long) i) << left_shift) + ccount;
- index++;
- }
- }
- }
- }
-
- public void incrementCount() {
- count++;
- }
-
- @Override
- public String toString() {
- return "Number: " + number + ", Count: " + count;
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupU.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupU.java
deleted file mode 100644
index 4e375ef..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/GroupU.java
+++ /dev/null
@@ -1,127 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import java.util.Arrays;
-
-import static java.lang.Math.pow;
-
-public class GroupU{
-
- public int[] number;
- public int count;
- public int[] count_array;
- public int range;
- public int unique_number;
- public int mask;
- public int left_shift;
- public long[] sorted_value_list; // cdf
- public int[] invert;
- public int[] invert2;
-
- public int getCount(long long1) {
- return ((int) (long1 & this.mask));
- }
- public int getUniqueValue(long long1) {
- return ((int) ((long1) >> this.left_shift));
- }
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
-
- public GroupU(int[] number, int count, int i) {
- this.number = number;
- this.count = count;
- this.range = (int) pow(2,i-1);
- this.count_array = new int[range];
- }
-
- public int[] getNumber() {
- return number;
- }
-
-
- public int getCount() {
- return count;
- }
-
- public void addNumber(int number) {
- this.number[this.count] = number;
- this.count++;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public void setCount_array(int max_){
- int k2_end = max_ - this.range;
- double par = this.range/(count*Math.log(this.count));
- this.left_shift = getBitWith(this.count);
- this.mask = (1 << left_shift) - 1;
- invert = new int[this.range+1];
- if (par > 3) {
- int[] value_list = new int[this.count];
- for (int i = 0; i < this.count; i++) {
- int value = this.number[i];
- count_array[k2_end - value]++;
- if (count_array[k2_end - value] == 1) {
- value_list[unique_number] = value;
- unique_number++;
- }
- }
- sorted_value_list = new long[unique_number];
- for (int i = 0; i < unique_number; i++) {
- int value = value_list[i];
- sorted_value_list[i] = (((long) (k2_end - value)) << left_shift) + count_array[k2_end - value];
- }
- Arrays.sort(sorted_value_list);
-
- int cdf_count = 0;
- for (int i = 0; i < unique_number; i++) {
- cdf_count += getCount(sorted_value_list[i]);
- sorted_value_list[i] = (((long) getUniqueValue(sorted_value_list[i])) << left_shift) + cdf_count;//new_value_list[i]
- }
- }else {
- int[] hash = new int[this.range];
- for (int i = 0; i < this.count; i++) {
- int value = this.number[i];
- if (hash[k2_end - value] == 0) {
- this.unique_number++;
- }
- hash[k2_end - value]++;
- }
- sorted_value_list = new long[unique_number];
- int ccount = 0;
- int index = 0;
- for (int i = 0; i < this.range; i++) {
- if (hash[i] > 0) {
- ccount += hash[i];
- sorted_value_list[index] = (((long) i) << left_shift) + ccount;
- index++;
- }
- }
- }
- }
-
- public void setinvert(){
- int count = 0;
- for (int i = 0; i < this.range; i++) {
- invert[i] = count;
- if (count < unique_number && getUniqueValue(sorted_value_list[count]) == i){
- count ++;
- }
- }
- invert[this.range] = count;
- }
-
- public void incrementCount() {
- count++;
- }
-
- @Override
- public String toString() {
- return "Number: " + number + ", Count: " + count;
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/PFORTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/PFORTest.java
deleted file mode 100644
index a695fbd..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/PFORTest.java
+++ /dev/null
@@ -1,1063 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-import me.lemire.integercompression.*;
-import org.jetbrains.annotations.NotNull;
-
-import java.io.*;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.*;
-
-import static org.apache.iotdb.tsfile.encoding.myPFOR.compressOneBlockOpt;
-import static org.apache.iotdb.tsfile.encoding.myPFOR.decompressOneBlock;
-
-public class PFORTest {
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- ArrayList<Integer> min_delta) {
- int[] ts_block_delta = new int[ts_block.length - 1];
-
-
- int value_delta_min = Integer.MAX_VALUE;
- for (int i = 1; i < ts_block.length; i++) {
-
- int epsilon_v = ts_block[i] - ts_block[i - 1];
-
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
-
- }
- min_delta.add(ts_block[0]);
- min_delta.add(value_delta_min);
-
- for (int i = 1; i < ts_block.length; i++) {
- int epsilon_v = ts_block[i] - value_delta_min - ts_block[i - 1];
- ts_block_delta[i - 1] = epsilon_v;
- }
- return ts_block_delta;
- }
-
- public static int zigzag(int num) {
- if (num < 0) return ((-num) << 1) - 1;
- else return num << 1;
- }
-
- public static int deZigzag(int num) {
- if (num % 2 == 0) return num >> 1;
- else return -((num + 1) >> 1);
- }
-
- public static int[] getAbsDeltaTsBlockSPRINTZ(
- int[] ts_block,
- ArrayList<Integer> min_delta) {
- int[] ts_block_delta = new int[ts_block.length - 1];
-
- for (int i = 1; i < ts_block.length; i++) {
-
- int epsilon_v = ts_block[i] - ts_block[i - 1];
- epsilon_v = zigzag(epsilon_v);
- ts_block_delta[i - 1] = epsilon_v;
-
- }
- min_delta.add(ts_block[0]);
-
- return ts_block_delta;
- }
-
- public static ArrayList<Integer> getAbsDeltaTsBlockRLE(
- int[] ts_block,
- ArrayList<Integer> min_delta,
- ArrayList<Integer> repeat_count) {
- ArrayList<Integer> ts_block_delta = new ArrayList<>();
-
- int value_delta_min = Integer.MAX_VALUE;
- for (Integer integer : ts_block) {
- if (integer < value_delta_min) value_delta_min = integer;
- }
- int repeat_i = 0;
- int pre_delta = ts_block[0] - value_delta_min;
- int pre_count = 1;
- min_delta.add(value_delta_min);
- int block_size = ts_block.length;
-
- for (int i = 1; i < block_size; i++) {
- int delta = ts_block[i] - value_delta_min;
- if (delta == pre_delta) {
- pre_count++;
- } else {
- if (pre_count > 7) {
- repeat_count.add(repeat_i); // index_repeat
- repeat_count.add(pre_count); // repeat_count
- ts_block_delta.add(pre_delta);
- } else {
- for (int j = 0; j < pre_count; j++)
- ts_block_delta.add(pre_delta);
- }
- pre_count = 1;
- repeat_i = i;
- }
- pre_delta = delta;
- }
- for (int j = 0; j < pre_count; j++)
- ts_block_delta.add(pre_delta);
- return ts_block_delta;
- }
-
-
- public static void main(@NotNull String[] args) throws IOException {
- //String parent_dir = "iotdb/iotdb-core/tsfile/src/test/resources/"; // your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/pfor_ratio/";
- //String output_parent_dir = parent_dir + "pfor_ratio/";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<String> encoding_list = new ArrayList<>();
- ArrayList<String> encoding_list0 = new ArrayList<>();
- int blocksize = 2048;
-
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- encoding_list0.add("TS_2DIFF");
- encoding_list0.add("RLE");
- encoding_list0.add("SPRINTZ");
-
- encoding_list.add("NEWPFOR");
- encoding_list.add("OPTPFOR");
- encoding_list.add("FASTPFOR");
- encoding_list.add("PFOR");
-
- for (String item : dataset_name) {
- input_path_list.add(input_parent_dir + item);
- }
-
- output_path_list.add(output_parent_dir + "CS-Sensors_ratio.csv"); // 0
-
- output_path_list.add(output_parent_dir + "Metro-Traffic_ratio.csv");// 1
-
- output_path_list.add(output_parent_dir + "USGS-Earthquakes_ratio.csv");// 2
-
- output_path_list.add(output_parent_dir + "YZ-Electricity_ratio.csv"); // 3
-
- output_path_list.add(output_parent_dir + "GW-Magnetic_ratio.csv"); //4
-
- output_path_list.add(output_parent_dir + "TY-Fuel_ratio.csv");//5
-
- output_path_list.add(output_parent_dir + "Cyber-Vehicle_ratio.csv"); //6
-
- output_path_list.add(output_parent_dir + "Vehicle-Charge_ratio.csv");//7
-
- output_path_list.add(output_parent_dir + "Nifty-Stocks_ratio.csv");//8
-
- output_path_list.add(output_parent_dir + "TH-Climate_ratio.csv");//9
-
- output_path_list.add(output_parent_dir + "TY-Transport_ratio.csv");//10
-
- output_path_list.add(output_parent_dir + "EPM-Education_ratio.csv");//11
-
-// for (int file_i = 8; file_i < 9; file_i++) {
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
- String inputPath = input_path_list.get(file_i);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
- String[] head = {
- "Input Direction",
- "Column Index",
- "Encoding Algorithm",
- "Compress Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Compress Time",
- "Uncompress Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
- int fileRepeat;
- ArrayList<Integer> columnIndexes = new ArrayList<>(); // set the column indexes of compressed
- for (int i = 0; i < 2; i++) {
- columnIndexes.add(i, i);
- }
- int points = 0;
- IntegerCODEC codec = new Composition(new JustCopy(), new VariableByte());
- IntWrapper inputoffset;
- IntWrapper outputoffset;
- int[] compressed;
- for (String encoding : encoding_list) {
- if (Objects.equals(encoding, "FASTPFOR")) {
- codec = new Composition(new FastPFOR(), new VariableByte());
- } else if (Objects.equals(encoding, "NEWPFOR")) {
- codec = new Composition(new NewPFDS16(), new VariableByte());
- } else if (Objects.equals(encoding, "OPTPFOR")) {
- codec = new Composition(new OptPFDS16(), new VariableByte());
- } else if (Objects.equals(encoding, "PFOR")) {
- for (File f : tempList) {
- for (String encoding0 : encoding_list0) {
- fileRepeat = 0;
- double final_ratio = 0;
- int final_compressed_size = 0;
- System.out.println(f);
- long encodeTime = 0;
- long decodeTime = 0;
- while (fileRepeat < 1) {
- fileRepeat += 1;
- InputStream inputStream = Files.newInputStream(f.toPath());
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<String> data = new ArrayList<>();
- for (int index : columnIndexes) {
- if (index == 0) {
- continue;
- }
- loader.readHeaders();
- data.clear();
- while (loader.readRecord()) {
- String v = loader.getValues()[index];
- data.add(v);
- }
- points = data.size();
- inputStream.close();
-
- int[] origin = new int[data.size()];
- int i = 0;
- for (String value : data) {
- origin[i] = Integer.parseInt(value);
- i++;
- }
- switch (encoding0) {
- case "TS_2DIFF": {
- //TS_2DIFF+PFOR
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- int[] new_slice = getAbsDeltaTsBlock(slice, min);
- int[] sliceoutBlock = compressOneBlockOpt(new_slice, new_slice.length);
- outBlock.add(sliceoutBlock.length);
- outBlock.add(endindex - offset - 1);
- for (i = 0; i < sliceoutBlock.length; i++) {
- outBlock.add(sliceoutBlock[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- int[] sliceoutBlock = compressOneBlockOpt(minList, minList.length);
- for (i = sliceoutBlock.length - 1; i >= 0; i--) {
- outBlock.add(0, sliceoutBlock[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, sliceoutBlock.length);
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- decompressOneBlock(new_min, min_sliceBlock, min_origin_size);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceUncompressed = new int[origin_size];
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- decompressOneBlock(sliceUncompressed, sliceBlock, origin_size);
- uncompressed[j] = new_min[k];
- j++;
- k++;
- for (i = 0; i < origin_size; i++) {
- uncompressed[j] = sliceUncompressed[i] + new_min[k] + uncompressed[j - 1];
- j++;
- }
- k++;
- offset = endindex;
- }
- decodeTime += System.nanoTime() - s;
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- break;
- }
- case "SPRINTZ": {
- //TS_2DIFF+SPRINTZ
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- int[] new_slice = getAbsDeltaTsBlockSPRINTZ(slice, min);
- int[] sliceoutBlock = compressOneBlockOpt(new_slice, new_slice.length);
- outBlock.add(sliceoutBlock.length);
- outBlock.add(endindex - offset - 1);
- for (i = 0; i < sliceoutBlock.length; i++) {
- outBlock.add(sliceoutBlock[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- int[] sliceoutBlock = compressOneBlockOpt(minList, minList.length);
- for (i = sliceoutBlock.length - 1; i >= 0; i--) {
- outBlock.add(0, sliceoutBlock[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, sliceoutBlock.length);
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- decompressOneBlock(new_min, min_sliceBlock, min_origin_size);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceUncompressed = new int[origin_size];
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- decompressOneBlock(sliceUncompressed, sliceBlock, origin_size);
-
- uncompressed[j] = new_min[k];
- j++;
- k++;
- for (i = 0; i < origin_size; i++) {
- uncompressed[j] = deZigzag(sliceUncompressed[i]) + uncompressed[j - 1];
- j++;
- }
- offset = endindex;
- }
- decodeTime += System.nanoTime() - s;
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- break;
- }
- case "RLE": {
- //TS_2DIFF+RLE+++
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- ArrayList<Integer> repeat_list = new ArrayList<>();
- ArrayList<Integer> new_slice_list = getAbsDeltaTsBlockRLE(slice, min, repeat_list);
- int[] new_slice = new int[new_slice_list.size()];
- i = 0;
- for (int value : new_slice_list) {
- new_slice[i++] = value;
- }
- int[] repeat = new int[repeat_list.size()];
- i = 0;
- for (int value : repeat_list) {
- repeat[i++] = value;
- }
- int[] sliceoutBlock = compressOneBlockOpt(new_slice, new_slice.length);
- outBlock.add(sliceoutBlock.length);
- outBlock.add(new_slice.length);
- for (i = 0; i < sliceoutBlock.length; i++) {
- outBlock.add(sliceoutBlock[i]);
- }
- int[] sliceoutBlock2 = compressOneBlockOpt(repeat, repeat.length);
- outBlock.add(sliceoutBlock2.length);
- outBlock.add(repeat.length);
- for (i = 0; i < sliceoutBlock2.length; i++) {
- outBlock.add(sliceoutBlock2[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- int[] sliceoutBlock = compressOneBlockOpt(minList, minList.length);
- for (i = sliceoutBlock.length - 1; i >= 0; i--) {
- outBlock.add(0, sliceoutBlock[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, sliceoutBlock.length);
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- decompressOneBlock(new_min, min_sliceBlock, min_origin_size);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- //RLE---
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceUncompressed = new int[origin_size];
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- decompressOneBlock(sliceUncompressed, sliceBlock, origin_size);
- offset = endindex;
- slice_size = outBlock.get(offset);
- endindex = offset + 2 + slice_size;
- int origin_size2 = outBlock.get(offset + 1);
- int[] sliceUncompressed2 = new int[origin_size2];//
- int[] sliceBlock2 = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- decompressOneBlock(sliceUncompressed2, sliceBlock2, origin_size2);
-
- int index2 = 0;
- int index1;
- int fake_index = 0;
- for (index1 = 0; index1 < origin_size; index1++) {
- if (index2 < sliceUncompressed2.length && fake_index == sliceUncompressed2[index2]) {
- //repeat
- int times = sliceUncompressed2[index2 + 1];
- for (i = 0; i < times; i++) {
- uncompressed[j] = sliceUncompressed[index1] + new_min[k];
- j++;
- }
- fake_index += sliceUncompressed2[index2 + 1];
- index2 += 2;
- } else {
- uncompressed[j] = sliceUncompressed[index1] + new_min[k];
- j++;
- fake_index++;
- }
- }
- offset = endindex;
- k++;
- }
- decodeTime += System.nanoTime() - s;
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- break;
- }
- }
- }
- }
- String[] record = {
- f.toString(),
- "1",
- encoding0 + "+" + encoding,
- "NOCOMPRESSION",
- String.valueOf(1.0 * encodeTime / fileRepeat),
- String.valueOf(1.0 * decodeTime / fileRepeat),
- "0",
- "0",
- String.valueOf(points),
- String.valueOf(final_compressed_size / fileRepeat),
- String.valueOf(final_ratio / fileRepeat)
- };
-
- writer.writeRecord(record);
- }
- }
- continue;
- }
- for (File f : tempList) {
- for (String encoding0 : encoding_list0) {
- switch (encoding0) {
- case "TS_2DIFF": {
- fileRepeat = 0;
- double final_ratio = 0;
- int final_compressed_size = 0;
- System.out.println(f);
- long encodeTime = 0;
- long decodeTime = 0;
-
- while (fileRepeat < 1) {
- fileRepeat += 1;
- InputStream inputStream = Files.newInputStream(f.toPath());
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<String> data = new ArrayList<>();
-
- for (int index : columnIndexes) {
- if (index == 0) {
- continue;
- }
- loader.readHeaders();
- data.clear();
- while (loader.readRecord()) {
- String v = loader.getValues()[index];
- data.add(v);
- }
- points = data.size();
- inputStream.close();
-
- int[] origin = new int[data.size()];
- int i = 0;
- for (String value : data) {
- origin[i++] = Integer.parseInt(value);
- }
- // TS_2DIFF
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- int[] tmp = getAbsDeltaTsBlock(slice, min);
- compressed = new int[tmp.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(tmp, inputoffset, tmp.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- outBlock.add(compressed.length);
- outBlock.add(endindex - offset - 1);
- for (i = 0; i < compressed.length; i++) {
- outBlock.add(compressed[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- compressed = new int[minList.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(minList, inputoffset, minList.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- for (i = compressed.length - 1; i >= 0; i--) {
- outBlock.add(0, compressed[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, compressed.length);
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(min_sliceBlock, new IntWrapper(0), min_sliceBlock.length,
- new_min, recoffset);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- int[] recovered = new int[origin_size];
- recoffset = new IntWrapper(0);
- codec.uncompress(sliceBlock, new IntWrapper(0), sliceBlock.length,
- recovered, recoffset);
- uncompressed[j] = new_min[k];
- j++;
- k++;
- for (i = 0; i < origin_size; i++) {
- uncompressed[j] = recovered[i] + new_min[k] + uncompressed[j - 1];
- j++;
- }
- k++;
- offset = endindex;
- }
- decodeTime += System.nanoTime() - s;
-
-
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- }
- }
-
- String[] record = {
- f.toString(),
- "1",
- encoding0 + "+" + encoding,
- "NOCOMPRESSION",
- String.valueOf(1.0 * encodeTime / fileRepeat),
- String.valueOf(1.0 * decodeTime / fileRepeat),
- "0",
- "0",
- String.valueOf(points),
- String.valueOf(final_compressed_size / fileRepeat),
- String.valueOf(final_ratio / fileRepeat)
- };
- writer.writeRecord(record);
- break;
- }
- case "SPRINTZ": {
- fileRepeat = 0;
- double final_ratio = 0;
- int final_compressed_size = 0;
- System.out.println(f);
- long encodeTime = 0;
- long decodeTime = 0;
-
- while (fileRepeat < 1) {
- fileRepeat += 1;
- InputStream inputStream = Files.newInputStream(f.toPath());
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<String> data = new ArrayList<>();
-
- for (int index : columnIndexes) {
- if (index == 0) {
- continue;
- }
- loader.readHeaders();
- data.clear();
- while (loader.readRecord()) {
- String v = loader.getValues()[index];
- data.add(v);
- }
- points = data.size();
- inputStream.close();
-
- int[] origin = new int[data.size()];
- int i = 0;
- for (String value : data) {
- origin[i++] = Integer.parseInt(value);
- }
- // SPRINTZ
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- int[] tmp = getAbsDeltaTsBlockSPRINTZ(slice, min);
- compressed = new int[tmp.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(tmp, inputoffset, tmp.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- outBlock.add(compressed.length);
- outBlock.add(endindex - offset - 1);
- for (i = 0; i < compressed.length; i++) {
- outBlock.add(compressed[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- compressed = new int[minList.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(minList, inputoffset, minList.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- for (i = compressed.length - 1; i >= 0; i--) {
- outBlock.add(0, compressed[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, compressed.length);
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(min_sliceBlock, new IntWrapper(0), min_sliceBlock.length,
- new_min, recoffset);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- int[] recovered = new int[origin_size];
- recoffset = new IntWrapper(0);
- codec.uncompress(sliceBlock, new IntWrapper(0), sliceBlock.length,
- recovered, recoffset);
- uncompressed[j] = new_min[k];
- j++;
- k++;
- for (i = 0; i < origin_size; i++) {
- uncompressed[j] = deZigzag(recovered[i]) + uncompressed[j - 1];
- j++;
- }
- offset = endindex;
- }
- decodeTime += System.nanoTime() - s;
-
-
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- }
- }
-
- String[] record = {
- f.toString(),
- "1",
- encoding0 + "+" + encoding,
- "NOCOMPRESSION",
- String.valueOf(1.0 * encodeTime / fileRepeat),
- String.valueOf(1.0 * decodeTime / fileRepeat),
- "0",
- "0",
- String.valueOf(points),
- String.valueOf(final_compressed_size / fileRepeat),
- String.valueOf(final_ratio / fileRepeat)
- };
- writer.writeRecord(record);
- break;
- }
- case "RLE": {
- fileRepeat = 0;
- double final_ratio = 0;
- int final_compressed_size = 0;
- System.out.println(f);
- long encodeTime = 0;
- long decodeTime = 0;
-
- while (fileRepeat < 1) {
- fileRepeat += 1;
- InputStream inputStream = Files.newInputStream(f.toPath());
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<String> data = new ArrayList<>();
-
- for (int index : columnIndexes) {
- if (index == 0) {
- continue;
- }
- loader.readHeaders();
- data.clear();
- while (loader.readRecord()) {
- String v = loader.getValues()[index];
- data.add(v);
- }
- points = data.size();
- inputStream.close();
-
- int[] origin = new int[data.size()];
- int i = 0;
- for (String value : data) {
- origin[i++] = Integer.parseInt(value);
- }
- // RLE+++
- ArrayList<Integer> min = new ArrayList<>();
- ArrayList<Integer> outBlock = new ArrayList<>();
- long s = System.nanoTime();
- int offset = 0;
- while (offset < data.size()) {
- int endindex = offset + blocksize;
- if (endindex >= data.size()) {
- endindex = data.size();
- }
- int[] slice = Arrays.copyOfRange(origin, offset, endindex);
- ArrayList<Integer> repeat_list = new ArrayList<>();
- ArrayList<Integer> new_slice_list = getAbsDeltaTsBlockRLE(slice, min, repeat_list);
- int[] tmp = new int[new_slice_list.size()];
- i = 0;
- for (int value : new_slice_list) {
- tmp[i++] = value;
- }
- int[] repeat = new int[repeat_list.size()];
- i = 0;
- for (int value : repeat_list) {
- repeat[i++] = value;
- }
- compressed = new int[tmp.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(tmp, inputoffset, tmp.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- outBlock.add(compressed.length);
- outBlock.add(tmp.length);
- for (i = 0; i < compressed.length; i++) {
- outBlock.add(compressed[i]);
- }
- compressed = new int[repeat.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(repeat, inputoffset, repeat.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- outBlock.add(compressed.length);
- outBlock.add(repeat.length);
- for (i = 0; i < compressed.length; i++) {
- outBlock.add(compressed[i]);
- }
- offset = endindex;
- }
- int[] minList = new int[min.size()];
- i = 0;
- for (int value : min) {
- minList[i] = value;
- i++;
- }
- compressed = new int[minList.length + 1024];
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- codec.compress(minList, inputoffset, minList.length, compressed, outputoffset);
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- for (i = compressed.length - 1; i >= 0; i--) {
- outBlock.add(0, compressed[i]);
- }
- outBlock.add(0, minList.length);
- outBlock.add(0, compressed.length);
-
- int[] outBlock2 = new int[outBlock.size()];
- i = 0;
- for (int value : outBlock) {
- outBlock2[i] = value;
- i++;
- }
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- offset = 0;
- int[] uncompressed = new int[data.size()];
- int j = 0;
- int k = 0;
- int first_slice_size = outBlock.get(offset);
- int first_endindex = offset + 2 + first_slice_size;
- int min_origin_size = outBlock.get(offset + 1);
- int[] new_min = new int[min_origin_size];
- int[] min_sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, first_endindex);
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(min_sliceBlock, new IntWrapper(0), min_sliceBlock.length,
- new_min, recoffset);
- offset = first_endindex;
- while (offset != outBlock.size()) {
- //RLE---
- int slice_size = outBlock.get(offset);
- int endindex = offset + 2 + slice_size;
- int origin_size = outBlock.get(offset + 1);
- int[] sliceBlock = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- int[] recovered = new int[origin_size];
- recoffset = new IntWrapper(0);
- codec.uncompress(sliceBlock, new IntWrapper(0), sliceBlock.length,
- recovered, recoffset);
- offset = endindex;
- slice_size = outBlock.get(offset);
- endindex = offset + 2 + slice_size;
- int origin_size2 = outBlock.get(offset + 1);
- int[] recovered2 = new int[origin_size2];
- recoffset = new IntWrapper(0);
- int[] sliceBlock2 = Arrays.copyOfRange(outBlock2, offset + 2, endindex);
- codec.uncompress(sliceBlock2, new IntWrapper(0), sliceBlock2.length,
- recovered2, recoffset);
-
- int index2 = 0;
- int index1;
- int fake_index = 0;
- for (index1 = 0; index1 < origin_size; index1++) {
- if (index2 < recovered2.length && fake_index == recovered2[index2]) {
- //repeat
- int times = recovered2[index2 + 1];
- for (i = 0; i < times; i++) {
- uncompressed[j] = recovered[index1] + new_min[k];
- j++;
- }
- fake_index += recovered2[index2 + 1];
- index2 += 2;
- } else {
- uncompressed[j] = recovered[index1] + new_min[k];
- j++;
- fake_index++;
- }
- }
- k++;
- offset = endindex;
- }
- decodeTime += System.nanoTime() - s;
-
-
- final_ratio += 1.0 * outBlock.size() / origin.length;
- final_compressed_size += outBlock.size() * 4;
- }
- }
-
- String[] record = {
- f.toString(),
- "1",
- encoding0 + "+" + encoding,
- "NOCOMPRESSION",
- String.valueOf(1.0 * encodeTime / fileRepeat),
- String.valueOf(1.0 * decodeTime / fileRepeat),
- "0",
- "0",
- String.valueOf(points),
- String.valueOf(final_compressed_size / fileRepeat),
- String.valueOf(final_ratio / fileRepeat)
- };
- writer.writeRecord(record);
- break;
- }
- case "TS_2DIFF1": {
- fileRepeat = 0;
- double final_ratio = 0;
- int final_compressed_size = 0;
- System.out.println(f);
- long encodeTime = 0;
- long decodeTime = 0;
-
- while (fileRepeat < 1) {
- fileRepeat += 1;
- InputStream inputStream = Files.newInputStream(f.toPath());
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<String> data = new ArrayList<>();
-
- for (int index : columnIndexes) {
- if (index == 0) {
- continue;
- }
- loader.readHeaders();
- data.clear();
- while (loader.readRecord()) {
- String v = loader.getValues()[index];
- data.add(v);
- }
- points = data.size();
- inputStream.close();
-
- int[] origin = new int[data.size()];
- int i = 0;
- for (String value : data) {
- origin[i++] = Integer.parseInt(value);
- }
- // TS_2DIFF
- inputoffset = new IntWrapper(0);
- outputoffset = new IntWrapper(0);
- compressed = new int[origin.length + 1024];
- codec.compress(origin, inputoffset, origin.length, compressed, outputoffset);
- long s = System.nanoTime();
- compressed = Arrays.copyOf(compressed, outputoffset.intValue());
- encodeTime += System.nanoTime() - s;
- s = System.nanoTime();
- int[] uncompressed = new int[data.size()];
- IntWrapper recoffset = new IntWrapper(0);
- codec.uncompress(compressed, new IntWrapper(0), compressed.length,
- uncompressed, recoffset);
- decodeTime += System.nanoTime() - s;
-
-
- final_ratio += 1.0 * compressed.length / origin.length;
- final_compressed_size += compressed.length * 4;
- }
- }
-
- String[] record = {
- f.toString(),
- "1",
- encoding0 + "+" + encoding,
- "NOCOMPRESSION",
- String.valueOf(1.0 * encodeTime / fileRepeat),
- String.valueOf(1.0 * decodeTime / fileRepeat),
- "0",
- "0",
- String.valueOf(points),
- String.valueOf(final_compressed_size / fileRepeat),
- String.valueOf(final_ratio / fileRepeat)
- };
- writer.writeRecord(record);
- break;
- }
- }
- }
- }
- }
- writer.close();
- }
- }
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSBImproveTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSBImproveTest.java
new file mode 100644
index 0000000..8909dab
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSBImproveTest.java
@@ -0,0 +1,2430 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static java.lang.Math.pow;
+
+public class RLEBOSBImproveTest {
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i*block_size;
+ int end = i*block_size+remaining;
+ for (int j = base; j < end; j++) {
+
+ int integer = ts_block[j];
+ if (integer < value_delta_min) value_delta_min = integer;
+ if (integer > value_delta_max) {
+ value_delta_max = integer;
+ }
+ }
+ int pre_delta = ts_block[i*block_size]-value_delta_min;
+ int pre_count = 1;
+
+ min_delta[0]=(value_delta_min);
+ int repeat_i = 0;
+ int ts_block_delta_i = 0;
+ for (int j = base+1; j < end; j++) {
+ int delta = ts_block[j]-value_delta_min;
+ if(delta == pre_delta){
+ pre_count ++;
+ } else {
+ if(pre_count>7){
+ repeat_count.add(repeat_i);
+ repeat_count.add(pre_count);
+ ts_block_delta[ts_block_delta_i]=pre_delta;
+ ts_block_delta_i ++;
+ } else{
+ for (int k = 0; k < pre_count; k++){
+ ts_block_delta[ts_block_delta_i] =pre_delta;
+ ts_block_delta_i++;
+ }
+ }
+ pre_count =1;
+ repeat_i = j - i*block_size;
+ }
+ pre_delta = delta;
+
+ }
+ for (int j = 0; j < pre_count; j++){
+ ts_block_delta[ts_block_delta_i] =pre_delta;
+ ts_block_delta_i++;
+ }
+ min_delta[1]=(ts_block_delta_i);
+ min_delta[2]=(value_delta_max-value_delta_min);
+ int[] new_ts_block_delta = new int[ts_block_delta_i];
+ System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
+
+ return new_ts_block_delta;
+ }
+
+
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int init_block_size,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if (cur_value < final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+
+
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_x_l_plus);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int size = repeat_count.size();
+ intByte2Bytes(size,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (size != 0)
+ encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
+
+
+
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int left_bit_width = getBitWith(final_k_start_value);
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value );
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) {
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ encode_pos =encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
+ block_size = min_delta[1];
+
+
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = max_delta_value - (1<<gamma) + 1;
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+ int end_value_i = start_value_i + 1;
+
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 1; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value <= x_u_plus_pow_beta && k_end_value <= x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+
+
+ }
+ for(int beta = 1; beta <= gamma_max; beta++){
+ int pow_beta = 1<<beta;
+ int start_value_i = 0;
+ int end_value_i = start_value_i+1;
+
+ for (; start_value_i < unique_value_count-1; start_value_i++) {
+ long x_l_minusL = sorted_value_list[start_value_i];
+ int x_l_minus = getUniqueValue(x_l_minusL, left_shift) ;
+ int x_l_plus = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+ int x_u_plus_pow_beta = pow_beta+x_l_plus;
+ if(x_u_plus_pow_beta > max_delta_value) break;
+
+
+
+ cur_k1 = getCount(x_l_minusL,mask);
+ int lower_outlier_cost = cur_k1 * getBitWith(x_l_minus);
+
+ while ( end_value_i < unique_value_count) {
+// if(beta==3 && end_value_i==22)
+// {
+// System.out.println(x_l_minus);
+// System.out.println(x_l_plus);
+// }
+
+ int x_u_minus = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus < x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta){
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus - x_l_plus);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = x_l_minus;
+ final_x_l_plus = x_l_plus;
+ final_k_end_value = x_u_plus;
+ final_x_u_minus = x_u_minus;
+ }
+ break;
+ }
+// else if (x_u_minus >= x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta) {
+// break;
+// }
+
+ end_value_i++;
+ }
+ }
+
+ }
+
+ encode_pos = BOSEncodeBits(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ ArrayList<Integer> repeat_count_result = new ArrayList<>();
+ repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = repeat_count_result.get(0);
+
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+
+ if (final_alpha == 0) { // 0
+ int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
+// System.out.println("bitmap_bytes:" + bitmap_bytes);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < cur_block_size) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+
+ }
+
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, cur_block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+ int cur_i = 0;
+ int repeat_i = 0;
+ for (int i = 0; i < cur_block_size; i++) {
+
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ }
+ pre_v = current_delta + min_delta;
+ if (repeat_i < count_size) {
+ if (cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ repeat_i += 2;
+ }else {
+ cur_i++;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ }
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list,block_size, block_size,value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoder(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int init_block_size,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int size = repeat_count.size();
+ intByte2Bytes(size,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ if (size != 0){
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int repeat_count_v:repeat_count){
+ encode_pos = EncodeBits(repeat_count_v, bit_width_init, encode_pos, cur_byte, bit_index_list);
+ }
+ if(bit_index_list[0] != 8){
+ bit_index_list[0] = 8;
+ encode_pos ++;
+ }
+ }
+// encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
+
+// int2Bytes(min_delta[1],encode_pos,cur_byte);
+// encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+// System.out.println(encode_pos);
+ return encode_pos;
+
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
+ block_size = min_delta[1];
+
+
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = max_delta_value - (1<<gamma) + 1;
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+ int end_value_i = start_value_i + 1;
+
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 1; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value <= x_u_plus_pow_beta && k_end_value <= x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+
+
+ }
+ for(int beta = 0; beta <= gamma_max; beta++){
+ int pow_beta = 1<<beta;
+ int start_value_i = 0;
+ int end_value_i = start_value_i+1;
+
+ for (; start_value_i < unique_value_count-1; start_value_i++) {
+ long x_l_minusL = sorted_value_list[start_value_i];
+ int x_l_minus = getUniqueValue(x_l_minusL, left_shift) ;
+ int x_l_plus = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+ int x_u_plus_pow_beta = pow_beta+x_l_plus;
+ if(x_u_plus_pow_beta > max_delta_value) break;
+
+
+
+ cur_k1 = getCount(x_l_minusL,mask);
+ int lower_outlier_cost = cur_k1 * getBitWith(x_l_minus);
+
+ while ( end_value_i < unique_value_count) {
+// if(beta==3 && end_value_i==22)
+// {
+// System.out.println(x_l_minus);
+// System.out.println(x_l_plus);
+// }
+
+ int x_u_minus = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus < x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta){
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus - x_l_plus);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = x_l_minus;
+ final_x_l_plus = x_l_plus;
+ final_k_end_value = x_u_plus;
+ final_x_u_minus = x_u_minus;
+ }
+ break;
+ }
+// else if (x_u_minus >= x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta) {
+// break;
+// }
+
+ end_value_i++;
+ }
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+// int value0 = bytes2Integer(encoded, decode_pos, 4);
+// decode_pos += 4;
+// value_list[value_pos_arr[0]] =value0;
+// value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+// ArrayList<Integer> repeat_count = new ArrayList<>();
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int i = 0;i<count_size;i++){
+ int repeat_count_v = DecodeBits(encoded, bit_width_init, decode_list);
+ repeat_count.add(repeat_count_v);
+ }
+
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+// repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = decode_list[0];
+// decode_list[1]= 8;
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+
+
+ int pre_v;
+ int cur_i = 0;
+ int repeat_i = 0;
+ if(k1==0 && k2==0){
+// int pre_v = value0;
+ decode_list[0] = decode_pos;
+ decode_list[1]= 8;
+ for (int i = 0; i < cur_block_size; i++) {
+ pre_v = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+// value_list[value_pos_arr[0]++] = pre_v;
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ }
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+
+// value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < cur_block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+// int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < cur_block_size; i++) {
+// int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ pre_v = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ pre_v = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ pre_v = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ // Update the cumulative value and store it
+// pre_v += deZigzag(currentDelta);
+// value_list[valuePos++] = pre_v;
+ }
+// value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+// System.out.println(k);
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
+// System.out.println(decode_pos);
+ }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
+ }
+ }
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/rle_bos_b";
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 8; file_i < 9; file_i++) {
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+
+
+ }
+ }
+
+ @Test
+ public void BOSOUOExpTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/exp_b";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+
+ dataset_name.add("Synthetic_Exp_100");
+ dataset_name.add("Synthetic_Exp_1000");
+ dataset_name.add("Synthetic_Exp_10000");
+ dataset_name.add("Synthetic_Exp_100000");
+ dataset_name.add("Synthetic_Exp_1000000");
+ dataset_name.add("Synthetic_Normal_100");
+ dataset_name.add("Synthetic_Normal_1000");
+ dataset_name.add("Synthetic_Normal_10000");
+ dataset_name.add("Synthetic_Normal_100000");
+ dataset_name.add("Synthetic_Normal_1000000");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/Exp_100.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Exp_1000.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Exp_10000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_100000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_1000000.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_100.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_1000.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Normal_10000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_100000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_1000000.csv");//5
+// dataset_block_size.add(2048);
+
+ int repeatTime2 = 100;
+// for (int file_i = 8; file_i < 9; file_i++) {
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ if(f.toString().contains(".DS")) continue;
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+
+
+ }
+ }
+ @Test
+ public void BOSImproveDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/rle_bos_b";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 4; file_i < 5; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSImproveEncodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/rle_bos_b_improve";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 4; file_i < 5; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSHTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSHTest.java
deleted file mode 100644
index cbd88b1..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSHTest.java
+++ /dev/null
@@ -1,1559 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-
-import static java.lang.Math.*;
-
-public class RLEBOSHTest {
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
-
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
-
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
-
- public static void int2Bytes(int integer, int encode_pos, byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos + 1] = (byte) (integer >> 16);
- cur_byte[encode_pos + 2] = (byte) (integer >> 8);
- cur_byte[encode_pos + 3] = (byte) (integer);
- }
-
- public static void intByte2Bytes(int integer, int encode_pos, byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos, byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos + 1] = (byte) (integer >> 16);
- cur_byte[encode_pos + 2] = (byte) (integer >> 8);
- cur_byte[encode_pos + 3] = (byte) (integer);
- }
-
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx) << (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos++;
- bufIdx++;
- if (bufIdx >= width) {
- return;
- }
- }
- }
-
- }
-
- public static void unpack8Values(byte[] encoded, int offset, int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width, int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size() - start) / 8;
- for (int i = 0; i < block_num; i++) {
- pack8Values(numbers, start + i * 8, bit_width, encode_pos, encoded_result);
- encode_pos += bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values(encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
- }
- return result_list;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- ArrayList<Integer> min_delta,
- ArrayList<Integer> repeat_count) {
- int[] ts_block_delta = new int[ts_block.length];
-
-
- int value_delta_min = Integer.MAX_VALUE;
- for (Integer integer : ts_block) {
- if (integer < value_delta_min) value_delta_min = integer;
- }
- int repeat_i = 0;
- int pre_delta = ts_block[0] - value_delta_min;
- int pre_count = 1;
- min_delta.add(value_delta_min);
- int block_size = ts_block.length;
-
- int ts_block_delta_i = 0;
- for (int i = 1; i < block_size; i++) {
- int delta = ts_block[i] - value_delta_min;
- if (delta == pre_delta) {
- pre_count++;
- } else {
- if (pre_count > 7) {
- repeat_count.add(repeat_i);
- repeat_count.add(pre_count);
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- } else {
- for (int j = 0; j < pre_count; j++) {
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- }
-
- }
- pre_count = 1;
- repeat_i = i;
- }
- pre_delta = delta;
- }
- for (int j = 0; j < pre_count; j++) {
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- }
- min_delta.add(ts_block_delta_i);
- int[] new_ts_block_delta = new int[ts_block_delta_i];
- System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
-
- return new_ts_block_delta;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta,
- ArrayList<Integer> repeat_count) {
- int[] ts_block_delta = new int[remaining];
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- int base = i * block_size;
- int end = i * block_size + remaining;
- for (int j = base; j < end; j++) {
-
- int integer = ts_block[j];
- if (integer < value_delta_min) value_delta_min = integer;
- if (integer > value_delta_max) {
- value_delta_max = integer;
- }
- }
- int pre_delta = ts_block[i * block_size] - value_delta_min;
- int pre_count = 1;
-
- min_delta[0] = (value_delta_min);
- int repeat_i = 0;
- int ts_block_delta_i = 0;
- for (int j = base + 1; j < end; j++) {
- int delta = ts_block[j] - value_delta_min;
- if (delta == pre_delta) {
- pre_count++;
- } else {
- if (pre_count > 7) {
- repeat_count.add(repeat_i);
- repeat_count.add(pre_count);
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- } else {
- for (int k = 0; k < pre_count; k++) {
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- }
- }
- pre_count = 1;
- repeat_i = j - i * block_size;
- }
- pre_delta = delta;
-
- }
- for (int j = 0; j < pre_count; j++) {
- ts_block_delta[ts_block_delta_i] = pre_delta;
- ts_block_delta_i++;
- }
- min_delta[1] = (ts_block_delta_i);
- min_delta[2] = (value_delta_max - value_delta_min);
- int[] new_ts_block_delta = new int[ts_block_delta_i];
- System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
-
- return new_ts_block_delta;
- }
-
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining, encode_pos, encoded_result);
- encode_pos += 4;
-
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining, encode_pos, encoded_result);
- encode_pos += 4;
- return encode_pos;
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
-
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int init_block_size,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- ArrayList<Integer> repeat_count,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if (cur_value < final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
-
-
- k1++;
-
-
- } else if (cur_value > final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value - 1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
-
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
- int final_alpha = ((k1 + k2) * getBitWith(block_size - 1)) <= (block_size + k1 + k2) ? 1 : 0;
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
- int2Bytes(k_byte, encode_pos, cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0], encode_pos, cur_byte);
- encode_pos += 4;
- int size = repeat_count.size();
- intByte2Bytes(size, encode_pos, cur_byte);
- encode_pos += 1;
-
- if (size != 0)
- encode_pos = encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size - 1), encode_pos, cur_byte);
-
-
- int2Bytes(final_k_start_value, encode_pos, cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value - 2);
- intByte2Bytes(bit_width_final, encode_pos, cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);
- intByte2Bytes(left_bit_width, encode_pos, cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width, encode_pos, cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) {
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i, encode_pos, cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size - 1), encode_pos, cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size - 1), encode_pos, cur_byte);
- }
- encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final, encode_pos, cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width, encode_pos, cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width, encode_pos, cur_byte);
- return encode_pos;
-
- }
-
-
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining, int encode_pos, byte[] cur_byte) {
-
- ArrayList<Integer> repeat_count = new ArrayList<>();
- int init_block_size = block_size;
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, init_block_size, remaining, min_delta, repeat_count);
- block_size = min_delta[1];
- int max_delta_value = min_delta[2];
-
-
-
-
- int boundary_index = (int) ((double) block_size / (double) (getBitWith(block_size-1)-1));
-
- int max_bit_width = getBitWith(max_delta_value) + 1;
- int[] lambda_list = {getBitWith(block_size-1),1};
- int lambda_0 = lambda_list[0];
-
- int count_x_min = 0;
- int count_x_max = 0;
- GroupL[] groupL = new GroupL[max_bit_width];
- GroupU[] groupU = new GroupU[max_bit_width];
- for (int i = 0; i < max_bit_width; i++) {
- int[] numbers = new int[block_size];
- int count = 0;
- groupL[i] = new GroupL(numbers, count, i);
- int[] numbersU = new int[block_size];
- groupU[i] = new GroupU(numbersU, count, i);
- }
- for (int value : ts_block_delta) {
- int alpha_i = getBitWith(value);
- if (value == 0) {
- count_x_min++;
- } else {
- groupL[alpha_i].addNumber(value); // (x_min+2^{alpha_i-1},x_min+2^{alpha_i})
- }
- int gamma_i = getBitWith(max_delta_value - value);
- if (value == max_delta_value) {
- count_x_max++;
- } else {
- groupU[gamma_i].addNumber(value);// (x_max-2^{gamma_i},x_max-2^{gamma_i-1})
- }
- }
- int[] alpha_list = new int[max_bit_width];
- int num_of_alpha_list = 0;
- int[] gamma_list = new int[max_bit_width];
- int num_of_gamma_list = 0;
- for (int i = 1; i < max_bit_width; i++) {
- if(groupL[i].count!=0){
- groupL[i].setCount_array();
- alpha_list[num_of_alpha_list] = i;
- num_of_alpha_list ++;
- }
- if(groupU[i].count!=0){
- groupU[i].setCount_array(max_delta_value);
- gamma_list[num_of_gamma_list] = i;
- num_of_gamma_list ++;
- groupU[i].setinvert();
- }
- }
-
-
- int final_k_start_value = -1;//getUniqueValue(sorted_value_list[0], left_shift);
- int final_k_end_value = max_delta_value + 1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
-
- int alpha_size = getBitWith(max(max_delta_value-1,0));
-
- if( num_of_alpha_list ==0 && num_of_gamma_list==0){
- encode_pos = BOSEncodeBits(ts_block_delta, init_block_size, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, repeat_count, encode_pos, cur_byte);
- return encode_pos;
- }
-
-
- // -------------x_min-------------------
- // lower: x_min
- // upper: x_max
- int k1 = count_x_min;
- int k1_start = 0;
- int k2 = count_x_max;
- int k2_end = max_delta_value;
- int cur_bits = Math.min((k1 + k2) * getBitWith(block_size - 1), block_size + k1 + k2);
- cur_bits += k1;
- cur_bits += k2;
- cur_bits += (block_size - k1 - k2) * getBitWith(k2_end - k1_start - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
- // lower: x_min
- // upper: x_max - 1, x_max - 2, x_max - 4 ...
- //(x_max - 2^{gamma}, x_max - 2^{gamma-1} ]
-
- for (int gamma_i = 0; gamma_i < num_of_gamma_list-1; gamma_i++) {
-
- int gamma = gamma_list[gamma_i];
-
- int gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma-1} gamma : 1 to alpha_size-1
-
- GroupU cur_group_gamma = groupU[gamma];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
- int cur_gamma_count = cur_group_gamma.count;
-
- for(int unique_i=0;unique_i<gamma_unique_number;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;//min_upper_outlier
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- k2 += cur_gamma_count;
-
- }
-
- // lower: x_min
- // k2_end = x_max - 2^{alpha_size-1}
- // upper: (0,x_max - 2^{alpha_size-1}]
-
- int gap_gamma = (int) pow(2,alpha_size-1);
- k2_end = max_delta_value - gap_gamma;
- GroupU cur_group_gamma = groupU[alpha_size];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
-
- for(int unique_i=0;unique_i<gamma_unique_number-1;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * alpha_size;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- // -------------x_min-------------------
-
-
-
-
- // [x_min+2^{alpha-1},x_min+2^{alpha}) alpha = 1 to alphasize - 1
- // -----------------[x_min+2^{alpha-1},x_min+2^{alpha})-------------------
- // alpha_size = getBitWith(max_delta_value);
-
- for (int alpha_i = 0; alpha_i < num_of_alpha_list-1; alpha_i++) {
-
- int alpha = alpha_list[alpha_i];
- int gap_alpha = (int) pow(2,alpha-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha];
-
-
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: x_max
-
- int alpha_value_count = cur_group_alpha.count;
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- // k1; count[x_min, x_min+2^{alpha-1}]
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int unique_i_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
- k2 = count_x_max;
- k2_end = max_delta_value;
- cur_bits = Math.min((unique_i_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_i_cur_k1 + k2);
- cur_bits += unique_i_cur_k1 * alpha;
- cur_bits += k2;
- if (unique_i_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_i_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
-
-
- int pow_2_alpha = (int) pow(2, alpha);
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
-
-
- int cur_k1 = k1; //count[x_min, x_min+2^{alpha-1}]
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- int gamma_size = (int) (log(max_delta_value - (int) pow(2, alpha)) / log(2));
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- int cur_k2 = k2; // count(x_max)
- int gamma_i = 0;
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- if(gamma > gamma_size){
- break;
- }
-
-
- // k1: count[x_min,x_min+2^{alpha-1}]
-
- gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma;
- cur_group_gamma = groupU[gamma]; // (x_max - 2^{gamma}, x_max - 2^{gamma-1})
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
-
- int pow_2_gamma = gap_gamma*2;
-
- // prop 5.7
-
- int cur_k1_start = k1_start + gap_alpha - 1;
- int cur_k2_end = k2_end - gap_gamma + 1;
- int cur_k2_start_of_lambda = cur_k2 + gamma_value_count;// count[x_max - 2^{gamma}+1, x_max]
- int cur_k1_start_of_lambda = k1 + alpha_value_count; // count[x_min, 2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_start_of_lambda + cur_k2_start_of_lambda) * lambda_0, block_size + cur_k1_start_of_lambda + cur_k2_start_of_lambda); // (lambda * (cur_cur_k1 + cur_k2_start_of_lambda) + zeta);
- cur_bits += (cur_k1_start_of_lambda * alpha);
- cur_bits += (cur_k2_start_of_lambda * gamma);
- cur_bits += ((block_size - cur_k1_start_of_lambda - cur_k2_start_of_lambda) * getBitWith(cur_k2_end - cur_k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = cur_k1_start;
- final_k_end_value = cur_k2_end;
- }
-
- // prop 5.4
- int prop_5_4_k1 = cur_k1 + alpha_value_count; //count[x_min, x_min+2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((prop_5_4_k1 + cur_k2) * lambda_0, block_size + prop_5_4_k1 + cur_k2); //(lambda * (prop_5_4_k1 + prop_5_4_k2) + zeta);
- cur_bits += (prop_5_4_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - prop_5_4_k1 - cur_k2) * getBitWith(k2_end - k1_start - gap_alpha - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + gap_alpha - 1;
- final_k_end_value = k2_end;
- }
-
- // prop 5.5
- int prop_5_5_k2 = cur_k2 + gamma_value_count; // count[x_max - 2^{gamma}+1,x_max}
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + prop_5_5_k2) * lambda_0, block_size + cur_k1 + prop_5_5_k2);// (lambda * (prop_5_5_k1 + prop_5_5_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (prop_5_5_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - prop_5_5_k2) * getBitWith(k2_end - gap_gamma - k1_start - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - gap_gamma + 1;
- }
-
- // prop 5.6
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + cur_k2) * lambda_0, block_size + cur_k1 + cur_k2);//(lambda * (prop_5_6_k1 + prop_5_6_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - cur_k2) * getBitWith(k2_end - k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
-
-
- if ((max_pow_2_lambda + 1) * pow_2_alpha + pow_2_gamma <= max_delta_value
- && pow_2_alpha + (max_pow_2_lambda + 1) * pow_2_gamma <= max_delta_value) {
-
-
- }
- else {
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}): k2_end - gap_gamma, k2_end
- // beta: getBitWidth(x_max - 2^{gamma} -(x_min+2^{alpha})) to getBitWidth(x_max - 2^{gamma-1}-(x_min+2^{alpha-1}) )
- int min_beta = getBitWith(max_delta_value-pow_2_gamma-pow_2_alpha);
- int max_beta = getBitWith(k2_end-k1_start);
-
-
-
- for(int beta = min_beta;beta <= max_beta;beta++) {
- int pow_2_beta = (int) pow(2, beta);
- if (beta >= alpha + max_lambda && beta <= gamma + 1 ){
- if(pow_2_beta + pow_2_alpha + gap_gamma >= max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
-
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- }else if (beta <= alpha + 1 && beta >= gamma + max_lambda) {
- if(pow_2_beta + gap_alpha + pow_2_gamma >= max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- } else if (beta <= alpha + 1 && beta <= gamma + 1) {
- if(pow_2_beta + gap_alpha + gap_gamma > max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- }else{
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
-
- }
-
- }
-
- cur_k2 = cur_k2 + gamma_value_count; // += count(x_max - 2^{gamma}, x_max - 2^{gamma-1})
-
- if(cur_k1+cur_k2>boundary_index){
- max_pow_2_lambda = (int) pow(2, 1);
- max_lambda = 1;
- }
-
- }
-
-
- k2 = cur_k2;
-
- // gamma: gamma_size+1 to ....
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- //k2_end = x_max - 2^{gamma-1} ; gamma : 1 to gamma_size
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]
-
-
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- cur_group_gamma = groupU[gamma];
- k2_end = max_delta_value - gap_gamma;
-
- if(k2_end <= k1_start){
- break;
- }
- int gamma_value_count = cur_group_gamma.count;
-
- gamma_unique_number = cur_group_gamma.unique_number;
- gamma_sorted = cur_group_gamma.sorted_value_list;
-
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int unique_j_cur_k1 = cur_k1+ cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- int x_u_i_end = Math.max(0,Math.min(k2_end - k1_start - x_l_i,gap_gamma));
- int unique_e = cur_group_gamma.invert[x_u_i_end];
- for (int unique_i = 0; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- int unique_i_cur_k2 = cur_k2+ cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- }
- cur_k2 += gamma_value_count;
- }
-
- k1 += alpha_value_count;
- }
- // -------------------[x_min+2^{alpha-1},x_min+2^{alpha})-----------------
-
-
- // [x_min+2^{alpha_size-1},x_max)
- // ----------------[x_min+2^{alpha_size-1},x_max)--------------------
-
- // x_min+2^{alpha_size-1} < x_max && x_min+2^{alpha_size} >= x_max
- // alpha_size - 1 < log(xmax) && alpha_size >= log(xmax)
- // alpha_size = \lceil log(x_max-1+1) \rceil
-
- alpha_size = alpha_list[num_of_alpha_list-1];
- int gap_alpha = (int) pow(2,alpha_size-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha_size];
- int alpha_value_count = cur_group_alpha.count;
-
-
- // lower: [x_min+2^{alpha_size-1},x_max)
- // ----------------- [x_min+2^{alpha_size-1},x_max)----------------------
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- int x_l_i_end = max_delta_value - k1_start; // [1, x_max - x_min+2^{alpha_size-1})
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- if (x_l_i >= x_l_i_end) {
- break;
- }
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
-
- // lower: [x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: x_max
- k2 = count_x_max;
- k2_end = max_delta_value;
-
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + k2);
- cur_bits += unique_j_cur_k1 * alpha_size;
- cur_bits += k2;
- if (unique_j_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
- // lower: (x_min+2^{alpha_size-1},x_max)
- // upper: (x_max - 2^{gamma}, x_max)
- // gamma: gamma_size+1 to ....
- // lower: (x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}] : k2_end - gap_gamma, k2_end
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- for (int gamma_i = 0; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma} ;
-
- if(k2_end<=k1_start){
- break;
- }
- cur_group_gamma = groupU[gamma];
-
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
- int pow_2_gamma = gap_gamma*2;
-
-
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- int x_u_i_end = k2_end - k1_start - x_l_i; // x_u_i>= k2_end - k1_start - x_l_i; break ;
-
- if(x_u_i_end <= 0 || x_l_i >= x_l_i_end){
- break;
- }
-
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- x_u_i_end = Math.min(x_u_i_end,gap_gamma - 1);
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
- for(int unique_i=0;unique_i<unique_e;unique_i++){
-
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha_size;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- }
-
-
-
-
- k2 += gamma_value_count;
-
-
-
- }
- // ------------------ [x_min+2^{alpha_size-1},x_max)--------------------
-
-
-
- encode_pos = BOSEncodeBits(ts_block_delta, init_block_size, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, repeat_count, encode_pos, cur_byte);
-
- return encode_pos;
- }
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all, encode_pos, encoded_result);
- encode_pos += 4;
-
- int block_num = length_all / block_size;
- int2Bytes(block_size, encode_pos, encoded_result);
- encode_pos += 4;
-
- for (int i = 0; i < block_num; i++) {
- encode_pos = BOSBlockEncoder(data, i, block_size, block_size, encode_pos, encoded_result);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- } else {
-
- int start = block_num * block_size;
- int remaining = length_all - start;
-
-
- encode_pos = BOSBlockEncoder(data, block_num, block_size, remaining, encode_pos, encoded_result);
-
- }
-
-
- return encode_pos;
- }
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int init_block_size, int block_size, int[] value_pos_arr) {
-
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int count_size = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> repeat_count = new ArrayList<>();
- if (count_size != 0) {
- ArrayList<Integer> repeat_count_result = new ArrayList<>();
- repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size - 1), count_size, repeat_count_result);
- decode_pos = repeat_count_result.get(0);
-
- }
-
- int cur_block_size = block_size;
- for (int i = 1; i < count_size; i += 2) {
- cur_block_size -= (repeat_count.get(i) - 1);
- }
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
- if (final_alpha == 0) { // 0
- int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < cur_block_size) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
-
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size - 1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
-
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size - 1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
-
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, cur_block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
-
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
- int cur_i = 0;
- int repeat_i = 0;
- for (int i = 0; i < cur_block_size; i++) {
-
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value + 1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value + 1;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value + 1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value + 1;
- normal_i++;
- }
- }
- pre_v = current_delta + min_delta;
- if (repeat_i < count_size) {
- if (cur_i == repeat_count.get(repeat_i)) {
- cur_i += (repeat_count.get(repeat_i + 1));
-
- for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- repeat_i += 2;
- } else {
- cur_i++;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- } else {
- cur_i++;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
- int[] value_list = new int[length_all + block_size];
- int[] value_pos_arr = new int[1];
-
- for (int k = 0; k < block_num; k++) {
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- BOSBlockDecoder(encoded, decode_pos, value_list, block_size, remain_length, value_pos_arr);
- }
- }
-
- public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
- String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/rle_pruning";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(256);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
- int repeatTime2 = 100;
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
-
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for (int i = 0; i < data2.size(); i++) {
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length * 4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
-
- String[] record = {
- f.toString(),
- "RLE+BOS-P",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-
- }
- writer.close();
-
-
- }
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSLTest.java
deleted file mode 100644
index 536e82d..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSLTest.java
+++ /dev/null
@@ -1,1214 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-import org.jetbrains.annotations.NotNull;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-
-import static java.lang.Math.pow;
-
-public class RLEBOSLTest {
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
-
- public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
- public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx)<< (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos ++;
- bufIdx++;
- if (bufIdx >= width) {
- return ;
- }
- }
- }
-
- }
-
- public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add ((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size()-start) / 8;
- for(int i=0;i<block_num;i++){
- pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
- encode_pos +=bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values( encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
- }
- return result_list;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- ArrayList<Integer> min_delta,
- ArrayList<Integer> repeat_count) {
- int[] ts_block_delta = new int[ts_block.length];
-
-
- int value_delta_min = Integer.MAX_VALUE;
- for (Integer integer : ts_block) {
- if (integer < value_delta_min) value_delta_min = integer;
- }
- int repeat_i = 0;
- int pre_delta = ts_block[0] - value_delta_min;
- int pre_count = 1;
- min_delta.add(value_delta_min);
- int block_size =ts_block.length;
-
- int ts_block_delta_i = 0;
- for (int i = 1; i < block_size; i++) {
- int delta = ts_block[i] - value_delta_min;
- if (delta == pre_delta) {
- pre_count++;
- } else {
- if (pre_count > 7) {
- repeat_count.add(repeat_i);
- repeat_count.add(pre_count);
- ts_block_delta[ts_block_delta_i]=pre_delta;
- ts_block_delta_i ++;
- } else {
- for (int j = 0; j < pre_count; j++){
- ts_block_delta[ts_block_delta_i] =pre_delta;
- ts_block_delta_i++;
- }
-
- }
- pre_count = 1;
- repeat_i = i;
- }
- pre_delta = delta;
- }
- for (int j = 0; j < pre_count; j++){
- ts_block_delta[ts_block_delta_i] =pre_delta;
- ts_block_delta_i++;
- }
- min_delta.add(ts_block_delta_i);
- int[] new_ts_block_delta = new int[ts_block_delta_i];
- System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
-
- return new_ts_block_delta;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta,
- ArrayList<Integer> repeat_count) {
- int[] ts_block_delta = new int[ts_block.length];
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- int base = i*block_size;
- int end = i*block_size+remaining;
- for (int j = base; j < end; j++) {
-
- int integer = ts_block[j];
- if (integer < value_delta_min) value_delta_min = integer;
- if (integer > value_delta_max) {
- value_delta_max = integer;
- }
- }
- int pre_delta = ts_block[i*block_size]-value_delta_min;
- int pre_count = 1;
-
- min_delta[0]=(value_delta_min);
- int repeat_i = 0;
- int ts_block_delta_i = 0;
- for (int j = base+1; j < end; j++) {
- int delta = ts_block[j]-value_delta_min;
- if(delta == pre_delta){
- pre_count ++;
- } else {
- if(pre_count>7){
- repeat_count.add(repeat_i);
- repeat_count.add(pre_count);
- ts_block_delta[ts_block_delta_i]=pre_delta;
- ts_block_delta_i ++;
- } else{
- for (int k = 0; k < pre_count; k++){
- ts_block_delta[ts_block_delta_i] =pre_delta;
- ts_block_delta_i++;
- }
- }
- pre_count =1;
- repeat_i = j - i*block_size;
- }
- pre_delta = delta;
-
- }
- for (int j = 0; j < pre_count; j++){
- ts_block_delta[ts_block_delta_i] =pre_delta;
- ts_block_delta_i++;
- }
- min_delta[1]=(ts_block_delta_i);
- min_delta[2]=(value_delta_max-value_delta_min);
- int[] new_ts_block_delta = new int[ts_block_delta_i];
- System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
-
- return new_ts_block_delta;
- }
-
-
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
-
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- return encode_pos;
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
-
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int init_block_size,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- ArrayList<Integer> repeat_count,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if (cur_value < final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
-
-
- k1++;
-
-
- } else if (cur_value > final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value-1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
-
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
- int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
- int2Bytes(k_byte,encode_pos,cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0],encode_pos,cur_byte);
- encode_pos += 4;
- int size = repeat_count.size();
- intByte2Bytes(size,encode_pos,cur_byte);
- encode_pos += 1;
-
- if (size != 0)
- encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
-
-
-
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) {
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- }
- encode_pos =encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
- return encode_pos;
-
- }
-
-
-
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
-
- ArrayList<Integer> repeat_count = new ArrayList<>();
- int init_block_size = block_size;
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
- block_size = min_delta[1];
-
-
- // RLE need to substitute
- int max_delta_value = min_delta[2];
- int max_bit = getBitWith(max_delta_value);
- int[] alpha_count_list = new int[max_bit+1];//count(xmin) count(xmin + 1) count(xmin + 2)... count(xmin + 8) ... count(xmin + 2^(i - 1))
- int[] alpha_box_count_list = new int[max_bit+1];// count(xmin, xmin + 1), count(xmin +1, xmin + 2)... count (xmin + 2^(i - 1), xmin + 2^i)
- int[] gamma_count_list = new int[max_bit+1];//count(xmax) count(xmax - 1)... count(xmax - 2^(i - 1))
- int[] gamma_box_count_list = new int[max_bit+1]; // count (xmax - 2^i, xmin - 2^(i-1))
- for(int value:ts_block_delta){
- if (value == 0){
- alpha_count_list[0]++;
- }
- else if (value == pow(2,getBitWith(value)-1)){
- alpha_count_list[getBitWith(value)]++;
- }else {
- alpha_box_count_list[getBitWith(value)]++;
- }
- if (value == max_delta_value){
- gamma_count_list[0]++;
- }
- else if (max_delta_value - value == pow(2,getBitWith(max_delta_value-value)-1)){
- gamma_count_list[getBitWith(max_delta_value-value)]++;
- }else {
- gamma_box_count_list[getBitWith(max_delta_value-value)]++;
- }
- }
-
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value -2) * (block_size));
-
- int cur_k1_close = alpha_count_list[0]; // lower: x_min
- int k_start_value_close = 0;
-
- int cur_bits = 0;
- int alpha_2_pow = 1;
- int gamma_size = (int) (Math.log(max_delta_value - alpha_2_pow - 1)/Math.log(2));
- int cur_k2_close = gamma_count_list[0];
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int k_end_value_close = max_delta_value - (int) pow(2,gamma);
-
- cur_k2_close += gamma_count_list[gamma+1];
- cur_k2_close += gamma_box_count_list[gamma];
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- int k_end_value_open = k_end_value_close + 1;
-
- //C4 k1 close k2 open
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
-
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
- }
-
- // lower: x_min +1
- // -------------------------------
- cur_k1_close += alpha_count_list[1]; // lower outliers: x_min +1
- cur_k1_close += alpha_box_count_list[0];
- k_start_value_close = 1;
- cur_k2_close = gamma_count_list[0];
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int k_end_value_close = max_delta_value - (int) pow(2,gamma);
-
- cur_k2_close += gamma_count_list[gamma+1];
- cur_k2_close += gamma_box_count_list[gamma];
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- int k_end_value_open = k_end_value_close + 1;
-
- //C4 k1 close k2 open
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
-
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
- }
- // lower: x_min + 2 ^alpha
- // -------------------------------
-
- for (int alpha = 1; alpha <= max_bit-1; alpha++) {
- //C1 k1 close k2 close
- k_start_value_close = (int) pow(2,alpha);//close
- int pow_2_alpha_i = alpha_count_list[alpha+1];
- int pow_2_alpha_i_1_to_2_alpha_i = alpha_box_count_list[alpha];
-
-
- cur_k1_close += pow_2_alpha_i;
- cur_k1_close += pow_2_alpha_i_1_to_2_alpha_i;
-
- int k_end_value_close;
-
- alpha_2_pow = (int)pow(2,alpha);
- gamma_size = (int) (Math.log(max_delta_value - alpha_2_pow - 1)/Math.log(2));
-
- // upper outliers: x_max
- cur_k2_close = gamma_count_list[0];
- k_end_value_close = max_delta_value;
-
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- int k_start_value_open = k_start_value_close - 1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha);
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- }
- // upper outliers: x_max -1
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- k_end_value_close = max_delta_value-1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- k_start_value_open = k_start_value_close - 1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha);
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- }
- // upper outliers: x_max -2^gamma
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int flag = 0;
- k_end_value_close = max_delta_value - (int) pow(2,gamma);
- cur_bits = 0;
- int pow_2_gamma_i = gamma_count_list[gamma+1];
- int pow_2_gamma_i_1_to_2_gamma_i = gamma_box_count_list[gamma];
-
- cur_k2_close += pow_2_gamma_i;
- cur_k2_close += pow_2_gamma_i_1_to_2_gamma_i;
-
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 1;
- }
-
- //C2 k1open k2open
- cur_bits = 0;
- cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- k_start_value_open = k_start_value_close - 1;
- int k_end_value_open = k_end_value_close + 1;
- cur_bits += Math.min((cur_k1_open + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_open);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * alpha;
- if (cur_k1_open + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_open - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 2;
- }
-
- //C3 k1 open k2 close
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * alpha;
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 3;
- }
-
- //C4 k1 close k2 open
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 4;
- }
- if(flag == 1){
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- } else if (flag == 2) {
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_open;
- } else if (flag == 3) {
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- } else if (flag == 4) {
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
-
- }
- }
-
-
-
- encode_pos = BOSEncodeBits(ts_block_delta,init_block_size, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta,repeat_count, encode_pos , cur_byte);
-
-
- return encode_pos;
- }
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all,encode_pos,encoded_result);
- encode_pos += 4;
-
- int block_num = length_all / block_size;
- int2Bytes(block_size,encode_pos,encoded_result);
- encode_pos+= 4;
-
- for (int i = 0; i < block_num; i++) {
- encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
-// System.out.println(encode_pos);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- } else {
-
- int start = block_num * block_size;
- int remaining = length_all-start;
-
-
- encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
-
-// int[] ts_block = new int[length_all-start];
-// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
-//
-//
-// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
-
- }
-
-
- return encode_pos;
- }
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
-
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int count_size = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> repeat_count = new ArrayList<>();
- if (count_size != 0) {
- ArrayList<Integer> repeat_count_result = new ArrayList<>();
- repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
- decode_pos = repeat_count_result.get(0);
-
- }
-
- int cur_block_size = block_size;
- for (int i = 1; i < count_size; i += 2) {
- cur_block_size -= (repeat_count.get(i) - 1);
- }
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
- if (final_alpha == 0) { // 0
- int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
-// System.out.println("bitmap_bytes:" + bitmap_bytes);
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < cur_block_size) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
-
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
-
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
-
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, cur_block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
-
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
- int cur_i = 0;
- int repeat_i = 0;
- for (int i = 0; i < cur_block_size; i++) {
-
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- }
- pre_v = current_delta + min_delta;
- if (repeat_i < count_size) {
- if (cur_i == repeat_count.get(repeat_i)) {
- cur_i += (repeat_count.get(repeat_i+1));
-
- for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- repeat_i += 2;
- }else {
- cur_i++;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- } else {
- cur_i++;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-// block_size--;
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
- int[] value_list = new int[length_all+block_size];
- int[] value_pos_arr = new int[1];
-
- for (int k = 0; k < block_num; k++) {
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list,block_size, block_size,value_pos_arr);
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- BOSBlockDecoder(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
- }
- }
-
- public static void main(@NotNull String[] args) throws IOException {
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/rle_bws";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(256);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
-// for (int file_i = 8; file_i < 9; file_i++) {
- int repeatTime2 = 100;
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
-
- loader.readHeaders();
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
-
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
-
- String[] record = {
- f.toString(),
- "RLE+BWS",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-
- }
- writer.close();
-
-
- }
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSMTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSMTest.java
new file mode 100644
index 0000000..8367368
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSMTest.java
@@ -0,0 +1,2417 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
+import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+import static java.lang.Math.*;
+import static org.apache.iotdb.tsfile.constant.TestConstant.random;
+
+public class RLEBOSMTest {
+
+ public static int findMedianImprove(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ return quickSelect2(arr, 0, n - 1, n / 2);
+ }
+
+ private static int quickSelect2(int[] arr, int left, int right, int k){
+ int pivotV=arr[left+random.nextInt(right-left+1)],tmpV;
+ int posEqual=left,posSmaller=left; // a[left,posEqual): = pivotV; a[posEqual,posSmaller): < pivotV
+ for(int i=left;i<=right;i++){
+ if(arr[i]==pivotV){
+ tmpV=arr[i];
+ arr[i]=arr[posSmaller];
+ arr[posSmaller]=arr[posEqual];
+ arr[posEqual]=tmpV;
+ posEqual++;
+ posSmaller++;
+ }else
+ if(arr[i]<pivotV){
+ tmpV=arr[posSmaller];
+ arr[posSmaller]=arr[i];
+ arr[i]=tmpV;
+ posSmaller++;
+ }
+ }
+ if(k+(posEqual-left)<=posSmaller-1)return quickSelect2(arr,posEqual,posSmaller-1,k+(posEqual-left));
+ else if(k<=posSmaller-1)return pivotV;
+ else return quickSelect2(arr,posSmaller,right,k);
+ }
+ public static int findMedian(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ return quickSelect(arr, 0, n - 1, n / 2);
+ }
+
+ private static int quickSelect(int[] arr, int left, int right, int k) {
+ if (left == right) {
+ return arr[left];
+ }
+
+ int pivotIndex = partition(arr, left, right);
+ if (k == pivotIndex) {
+ return arr[k];
+ } else if (k < pivotIndex) {
+ return quickSelect(arr, left, pivotIndex - 1, k);
+ } else {
+ return quickSelect(arr, pivotIndex + 1, right, k);
+ }
+ }
+
+ private static int partition(int[] arr, int left, int right) {
+ int pivot = arr[right];
+ int i = left;
+ for (int j = left; j < right; j++) {
+ if (arr[j] <= pivot) {
+ swap(arr, i, j);
+ i++;
+ }
+ }
+ swap(arr, i, right);
+ return i;
+ }
+
+ private static void swap(int[] arr, int i, int j) {
+ int temp = arr[i];
+ arr[i] = arr[j];
+ arr[j] = temp;
+ }
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count) {
+ int[] ts_block_delta = new int[remaining];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i*block_size;
+ int end = i*block_size+remaining;
+ for (int j = base; j < end; j++) {
+
+ int integer = ts_block[j];
+ if (integer < value_delta_min) value_delta_min = integer;
+ if (integer > value_delta_max) {
+ value_delta_max = integer;
+ }
+ }
+ int pre_delta = ts_block[i*block_size]-value_delta_min;
+ int pre_count = 1;
+
+ min_delta[0]=(value_delta_min);
+ int repeat_i = 0;
+ int ts_block_delta_i = 0;
+ for (int j = base+1; j < end; j++) {
+ int delta = ts_block[j]-value_delta_min;
+ if(delta == pre_delta){
+ pre_count ++;
+ } else {
+ if(pre_count>7){
+ repeat_count.add(repeat_i);
+ repeat_count.add(pre_count);
+ ts_block_delta[ts_block_delta_i]=pre_delta;
+ ts_block_delta_i ++;
+ } else{
+ for (int k = 0; k < pre_count; k++){
+ ts_block_delta[ts_block_delta_i] =pre_delta;
+ ts_block_delta_i++;
+ }
+ }
+ pre_count =1;
+ repeat_i = j - i*block_size;
+ }
+ pre_delta = delta;
+
+ }
+ for (int j = 0; j < pre_count; j++){
+ ts_block_delta[ts_block_delta_i] =pre_delta;
+ ts_block_delta_i++;
+ }
+ min_delta[1]=(ts_block_delta_i);
+ min_delta[2]=(value_delta_max-value_delta_min);
+ int[] new_ts_block_delta = new int[ts_block_delta_i];
+ System.arraycopy(ts_block_delta, 0, new_ts_block_delta, 0, ts_block_delta_i);
+
+ return new_ts_block_delta;
+ }
+
+
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static void addToArchiveCompression(SevenZOutputFile out, File file, String dir) {
+ String name = dir + File.separator + file.getName();
+ if(dir.equals(".")) {
+ name = file.getName();
+ }
+ if (file.isFile()){
+ SevenZArchiveEntry entry = null;
+ FileInputStream in = null;
+ try {
+ entry = out.createArchiveEntry(file, name);
+ out.putArchiveEntry(entry);
+ in = new FileInputStream(file);
+ byte[] b = new byte[1024];
+ int count = 0;
+ while ((count = in.read(b)) > 0) {
+ out.write(b, 0, count);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ out.closeArchiveEntry();
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ } else if (file.isDirectory()) {
+ File[] children = file.listFiles();
+ if (children != null){
+ for (File child : children){
+ addToArchiveCompression(out, child, name);
+ }
+ }
+ } else {
+ System.out.println(file.getName() + " is not supported");
+ }
+ }
+
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int init_block_size,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if (cur_value < final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+
+
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_x_l_plus);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int size = repeat_count.size();
+ intByte2Bytes(size,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (size != 0)
+ encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
+
+
+ int bit_width_final = 0;
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+
+ if(k1==0 && k2==0){
+ bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
+ }
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
+ encode_pos =encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
+ block_size = min_delta[1];
+ int max_delta_value = min_delta[2];
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedian(findMedianArray);
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value<=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+
+
+ }
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = (int) pow(2,beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+
+ encode_pos = BOSEncodeBits(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int init_block_size,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int size = repeat_count.size();
+ intByte2Bytes(size,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ if (size != 0){
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int repeat_count_v:repeat_count){
+ encode_pos = EncodeBits(repeat_count_v, bit_width_init, encode_pos, cur_byte, bit_index_list);
+ }
+ if(bit_index_list[0] != 8){
+ bit_index_list[0] = 8;
+ encode_pos ++;
+ }
+ }
+// encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
+
+// int2Bytes(min_delta[1],encode_pos,cur_byte);
+// encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+// System.out.println(encode_pos);
+ return encode_pos;
+
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
+ block_size = min_delta[1];
+ int max_delta_value = min_delta[2];
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedianImprove(findMedianArray);
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value<=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+
+
+ }
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = 1 << (beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ ArrayList<Integer> repeat_count_result = new ArrayList<>();
+ repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = repeat_count_result.get(0);
+
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+ int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ if(k1!=0 || k2 != 0){
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+ }else {
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+
+ decode_pos += 1;
+ }
+
+
+// System.out.println(bit_width_final);
+// System.out.println("cur_block_size" + cur_block_size);
+// System.out.println(k1);
+// System.out.println("k2"+k2);
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, cur_block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+ int cur_i = 0;
+ int repeat_i = 0;
+ for (int i = 0; i < cur_block_size; i++) {
+
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ }
+ pre_v = current_delta + min_delta;
+ if (repeat_i < count_size) {
+ if (cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ repeat_i += 2;
+ }else {
+ cur_i++;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ }
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+// System.out.println(k);
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list,block_size, block_size,value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoder(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+// int value0 = bytes2Integer(encoded, decode_pos, 4);
+// decode_pos += 4;
+// value_list[value_pos_arr[0]] =value0;
+// value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+// ArrayList<Integer> repeat_count = new ArrayList<>();
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int i = 0;i<count_size;i++){
+ int repeat_count_v = DecodeBits(encoded, bit_width_init, decode_list);
+ repeat_count.add(repeat_count_v);
+ }
+
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+// repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = decode_list[0];
+// decode_list[1]= 8;
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+
+
+ int pre_v;
+ int cur_i = 0;
+ int repeat_i = 0;
+ if(k1==0 && k2==0){
+// int pre_v = value0;
+ decode_list[0] = decode_pos;
+ decode_list[1]= 8;
+ for (int i = 0; i < cur_block_size; i++) {
+ pre_v = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+// value_list[value_pos_arr[0]++] = pre_v;
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ }
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+
+// value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < cur_block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+// int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < cur_block_size; i++) {
+// int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ pre_v = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ pre_v = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ pre_v = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ // Update the cumulative value and store it
+// pre_v += deZigzag(currentDelta);
+// value_list[valuePos++] = pre_v;
+ }
+// value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+// System.out.println(k);
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
+// System.out.println(decode_pos);
+ }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
+ }
+ }
+
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/rle_bos_m";
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 0; file_i < 1; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void ExpTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/exp_m";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("Synthetic_Exp_100");
+ dataset_name.add("Synthetic_Exp_1000");
+ dataset_name.add("Synthetic_Exp_10000");
+ dataset_name.add("Synthetic_Exp_100000");
+ dataset_name.add("Synthetic_Exp_1000000");
+ dataset_name.add("Synthetic_Normal_100");
+ dataset_name.add("Synthetic_Normal_1000");
+ dataset_name.add("Synthetic_Normal_10000");
+ dataset_name.add("Synthetic_Normal_100000");
+ dataset_name.add("Synthetic_Normal_1000000");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/Exp_100.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Exp_1000.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Exp_10000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_100000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_1000000.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_100.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_1000.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Normal_10000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_100000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_1000000.csv");//5
+// dataset_block_size.add(2048);
+
+ int repeatTime2 = 100;
+// for (int file_i = 8; file_i < 9; file_i++) {
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ if(f.toString().contains(".DS")) continue;
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void compressTest() throws IOException {
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/rle_m_c";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 1;
+// for (int file_i = 8; file_i < 9; file_i++) {
+ CompressionType[] compressList = {
+ CompressionType.LZ4,
+ CompressionType.LZMA2,
+ };
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+
+ for (CompressionType comp : compressList) {
+ double ratio = 0;
+ double compressed_size = 0;
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ byte[] compressed = compressor.compress(encoded_result);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ double ratio = 0;
+ double compressed_size = 0;
+ File outfile = new File(parent_dir + "icde0802/example.bin");
+
+ // 使用FileOutputStream将byte数组写入文件
+ try (FileOutputStream fos = new FileOutputStream(outfile)) {
+ fos.write(encoded_result);
+ } catch (IOException e2) {
+ // 处理可能的I/O异常
+ e2.printStackTrace();
+ }
+
+ File input = new File(parent_dir + "icde0802/example.bin");
+ File output = new File(parent_dir + "icde0802/example.7z");
+ SevenZOutputFile out = new SevenZOutputFile(output);
+
+ addToArchiveCompression(out, input, ".");
+ out.closeArchiveEntry();
+
+ long compressed = output.length();
+
+
+ // test compression ratio and compressed size
+ compressed_size += compressed;
+ double ratioTmp =
+ (double) compressed / (double) (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ "7ZIP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSImproveDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/rle_bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 4; file_i < 5; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSImproveEncodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/rle_bos_m_improve";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 4; file_i < 5; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSVTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSVTest.java
index 3cf9de6..223701c 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSVTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/RLEBOSVTest.java
@@ -2,6 +2,7 @@
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
@@ -11,7 +12,7 @@
import java.util.ArrayList;
import java.util.Arrays;
-import static java.lang.Math.pow;
+import static java.lang.Math.*;
public class RLEBOSVTest {
@@ -323,7 +324,9 @@
private static int BOSEncodeBits(int[] ts_block_delta,
int init_block_size,
int final_k_start_value,
+ int final_x_l_plus,
int final_k_end_value,
+ int final_x_u_minus,
int max_delta_value,
int[] min_delta,
ArrayList<Integer> repeat_count,
@@ -362,7 +365,7 @@
k1++;
- } else if (cur_value > final_k_end_value) {
+ } else if (cur_value >= final_k_end_value) {
final_right_outlier.add(cur_value - final_k_end_value);
final_right_outlier_index.add(i);
if (cur_index_bitmap_outlier_bits % 8 != 7) {
@@ -379,7 +382,7 @@
k2++;
} else {
- final_normal.add(cur_value - final_k_start_value-1);
+ final_normal.add(cur_value - final_x_l_plus);
index_bitmap_outlier <<= 1;
cur_index_bitmap_outlier_bits += 1;
}
@@ -415,13 +418,16 @@
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
intByte2Bytes(bit_width_final,encode_pos,cur_byte);
encode_pos += 1;
int left_bit_width = getBitWith(final_k_start_value);
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);
+ int right_bit_width = getBitWith(max_delta_value -final_k_end_value );
intByte2Bytes(left_bit_width,encode_pos,cur_byte);
encode_pos += 1;
intByte2Bytes(right_bit_width,encode_pos,cur_byte);
@@ -491,36 +497,81 @@
}
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value ) * (block_size));
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
- int start_value_size = unique_value_count;
- for (int start_value_i = 0; start_value_i < start_value_size; start_value_i++) {
- int k_start_value = getUniqueValue(sorted_value_list[start_value_i], left_shift) ;
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
- int cur_k1 = getCount(sorted_value_list[start_value_i],mask);
+ int k_start_value = -1; // x_l_minus
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
int k_end_value;
int cur_bits;
int cur_k2;
+ k_end_value = max_delta_value + 1;
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
cur_bits = 0;
cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
- if (cur_k1 != 0)
- cur_bits += cur_k1 * getBitWith(k_start_value);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
if (cur_k1 + cur_k2 != block_size)
- cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(k_end_value - k_start_value-2);
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
if (cur_k2 != 0)
cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
@@ -528,14 +579,17 @@
if (cur_bits < min_bits) {
min_bits = cur_bits;
final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
}
}
}
- encode_pos = BOSEncodeBits(ts_block_delta,init_block_size, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta,repeat_count, encode_pos , cur_byte);
+
+ encode_pos = BOSEncodeBits(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
return encode_pos;
}
@@ -585,6 +639,452 @@
return encode_pos;
}
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int init_block_size,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ ArrayList<Integer> repeat_count,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int size = repeat_count.size();
+ intByte2Bytes(size,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ if (size != 0){
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int repeat_count_v:repeat_count){
+ encode_pos = EncodeBits(repeat_count_v, bit_width_init, encode_pos, cur_byte, bit_index_list);
+ }
+ if(bit_index_list[0] != 8){
+ bit_index_list[0] = 8;
+ encode_pos ++;
+ }
+ }
+// encode_pos =encodeOutlier2Bytes(repeat_count, getBitWith(init_block_size-1),encode_pos,cur_byte);
+
+// int2Bytes(min_delta[1],encode_pos,cur_byte);
+// encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+// System.out.println(encode_pos);
+ return encode_pos;
+
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size,int remaining, int encode_pos , byte[] cur_byte) {
+
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ int init_block_size = block_size;
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block,block_i, init_block_size,remaining, min_delta, repeat_count);
+ block_size = min_delta[1];
+
+
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+
+ cur_bits = 0;
+ cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+ }
+
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta,init_block_size, final_k_start_value,final_x_l_plus, final_k_end_value,final_x_u_minus,
+ max_delta_value, min_delta,repeat_count, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
int k_byte = bytes2Integer(encoded, decode_pos, 4);
@@ -618,6 +1118,9 @@
int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
+ int final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
decode_pos += 1;
@@ -714,7 +1217,7 @@
int right_outlier_i = 0;
int normal_i = 0;
int pre_v;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
int cur_i = 0;
int repeat_i = 0;
@@ -803,10 +1306,321 @@
}
}
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list,int init_block_size, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+// int value0 = bytes2Integer(encoded, decode_pos, 4);
+// decode_pos += 4;
+// value_list[value_pos_arr[0]] =value0;
+// value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int count_size = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+// ArrayList<Integer> repeat_count = new ArrayList<>();
+ ArrayList<Integer> repeat_count = new ArrayList<>();
+ if (count_size != 0) {
+ int bit_width_init = getBitWith(init_block_size-1);
+ for(int i = 0;i<count_size;i++){
+ int repeat_count_v = DecodeBits(encoded, bit_width_init, decode_list);
+ repeat_count.add(repeat_count_v);
+ }
+
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+// repeat_count = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(init_block_size-1), count_size, repeat_count_result);
+ decode_pos = decode_list[0];
+// decode_list[1]= 8;
+ }
+
+ int cur_block_size = block_size;
+ for (int i = 1; i < count_size; i += 2) {
+ cur_block_size -= (repeat_count.get(i) - 1);
+ }
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+
+
+ int pre_v;
+ int cur_i = 0;
+ int repeat_i = 0;
+ if(k1==0 && k2==0){
+// int pre_v = value0;
+ decode_list[0] = decode_pos;
+ decode_list[1]= 8;
+ for (int i = 0; i < cur_block_size; i++) {
+ pre_v = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+// value_list[value_pos_arr[0]++] = pre_v;
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ }
+ if(decode_list[1] != 8){
+ decode_list[1] = 8;
+ decode_list[0] ++;
+ }
+
+// value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (cur_block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < cur_block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(cur_block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+// int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < cur_block_size; i++) {
+// int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ pre_v = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ pre_v = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ pre_v = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+ if (repeat_i < count_size && cur_i == repeat_count.get(repeat_i)) {
+ cur_i += (repeat_count.get(repeat_i+1));
+
+ for (int j = 0; j < repeat_count.get(repeat_i + 1); j++) {
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ repeat_i += 2;
+ } else {
+ cur_i++;
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ // Update the cumulative value and store it
+// pre_v += deZigzag(currentDelta);
+// value_list[valuePos++] = pre_v;
+ }
+// value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ int[] value_pos_arr = new int[1];
+
+ for (int k = 0; k < block_num; k++) {
+// System.out.println(k);
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size, block_size, value_pos_arr);
+// System.out.println(decode_pos);
+ }
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list,block_size, remain_length, value_pos_arr);
+ }
+ }
+
public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/rle_bos";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/rle_bos";
String input_parent_dir = parent_dir + "trans_data/";
ArrayList<String> input_path_list = new ArrayList<>();
ArrayList<String> output_path_list = new ArrayList<>();
@@ -931,7 +1745,7 @@
String[] record = {
f.toString(),
- "RLE+BOS-O",
+ "RLE+BOS-V",
String.valueOf(encodeTime),
String.valueOf(decodeTime),
String.valueOf(data1.size()),
@@ -947,4 +1761,285 @@
}
}
+ @Test
+ public void ExpTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/exp_v";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+
+ dataset_name.add("Synthetic_Exp_100");
+ dataset_name.add("Synthetic_Exp_1000");
+ dataset_name.add("Synthetic_Exp_10000");
+ dataset_name.add("Synthetic_Exp_100000");
+ dataset_name.add("Synthetic_Exp_1000000");
+ dataset_name.add("Synthetic_Normal_100");
+ dataset_name.add("Synthetic_Normal_1000");
+ dataset_name.add("Synthetic_Normal_10000");
+ dataset_name.add("Synthetic_Normal_100000");
+ dataset_name.add("Synthetic_Normal_1000000");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/Exp_100.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Exp_1000.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Exp_10000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_100000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_1000000.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_100.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_1000.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Normal_10000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_100000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_1000000.csv");//5
+// dataset_block_size.add(2048);
+
+ int repeatTime2 = 1000;
+// for (int file_i = 8; file_i < 9; file_i++) {
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-V",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+
+
+ }
+ }
+
+ @Test
+ public void BOSImproveTimeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/rle_bos_v";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 4; file_i < 5; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "RLE+BOS-V",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSBTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSBTest.java
new file mode 100644
index 0000000..2090235
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSBTest.java
@@ -0,0 +1,2150 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static java.lang.Math.pow;
+
+public class SPRINTZBOSBTest {
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int zigzag(int num) {
+ if (num < 0) return ((-num)<<1)-1;
+ else return num<<1;
+ }
+
+ public static int deZigzag(int num) {
+ if (num % 2 == 0) return num>>1;
+ else return -((num+1)>>1);
+ }
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+
+ // -----------------------------------------------------------------
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining-1];
+
+ int base = i*block_size+1;
+ int end = i*block_size+remaining;
+ min_delta[0]=ts_block[base-1];
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ for (int j = base; j < end; j++) {
+ int epsilon_v = ts_block[j] - ts_block[j - 1];
+ epsilon_v = zigzag(epsilon_v);
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ ts_block_delta[j-base] =epsilon_v;
+
+ }
+ for (int j = 0; j < remaining-1; j++) {
+ ts_block_delta[j] =ts_block_delta[j]-value_delta_min;
+
+ }
+ min_delta[1] = (value_delta_min);
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+
+ return ts_block_delta;
+ }
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_k_start_value-1);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte); // x0
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte); // x_min
+ encode_pos += 4;
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+// int2Bytes(final_k_start_value,encode_pos,cur_byte);
+// encode_pos += 4;
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = (int) (max_delta_value - pow(2, gamma) + 1);
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+ int end_value_i = start_value_i + 1;
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+ for(int beta = 1; beta < beta_max; beta++){
+ int x_u_plus_pow_beta = (int) (x_l_plus_value + pow(2,beta));
+
+ for (; end_value_i < unique_value_count; end_value_i++) {
+ long k_end_valueL = sorted_value_list[end_value_i-1];
+
+ x_u_minus_value = getUniqueValue(k_end_valueL, left_shift);
+ k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+ cur_bits = 0;
+ cur_k2 = block_size - getCount(k_end_valueL,mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+ } else if (x_u_minus_value >= x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 1; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value <= x_u_plus_pow_beta && k_end_value <= x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+
+
+ }
+
+ encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoder(data, i, block_size,block_size, encode_pos,encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+// int supple_length;
+// if (remaining_length % 8 == 0) {
+// supple_length = 1;
+// } else if (remaining_length % 8 == 1) {
+// supple_length = 0;
+// } else {
+// supple_length = 9 - remaining_length % 8;
+// }
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, supple_length, encode_pos,encoded_result);
+ }
+
+
+ return encode_pos;
+ }
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = max_delta_value - (1<< gamma) + 1;
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+
+
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 0; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value < x_u_plus_pow_beta && k_end_value < x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+//
+
+ }
+
+ for(int beta = 0; beta < gamma_max; beta++){
+
+ int pow_beta = 1<<beta;
+ int start_value_i = 0;
+ int end_value_i = start_value_i+1;
+
+ for (; start_value_i < unique_value_count-1; start_value_i++) {
+ long x_l_minusL = sorted_value_list[start_value_i];
+ int x_l_minus = getUniqueValue(x_l_minusL, left_shift) ;
+ int x_l_plus = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+ int x_u_plus_pow_beta = pow_beta+x_l_plus;
+ if(x_u_plus_pow_beta > max_delta_value) break;
+
+
+
+ cur_k1 = getCount(x_l_minusL,mask);
+ int lower_outlier_cost = cur_k1 * getBitWith(x_l_minus);
+
+ while ( end_value_i < unique_value_count) {
+// if(beta==3 && end_value_i==22)
+// {
+// System.out.println(x_l_minus);
+// System.out.println(x_l_plus);
+// }
+
+ int x_u_minus = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus < x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta){
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus - x_l_plus);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = x_l_minus;
+ final_x_l_plus = x_l_plus;
+ final_k_end_value = x_u_plus;
+ final_x_u_minus = x_u_minus;
+ }
+ break;
+ }
+// else if (x_u_minus >= x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta) {
+// break;
+// }
+
+ end_value_i++;
+ }
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size,block_size, encode_pos,encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+// int supple_length;
+// if (remaining_length % 8 == 0) {
+// supple_length = 1;
+// } else if (remaining_length % 8 == 1) {
+// supple_length = 0;
+// } else {
+// supple_length = 9 - remaining_length % 8;
+// }
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, supple_length, encode_pos,encoded_result);
+ }
+
+
+ return encode_pos;
+ }
+
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+
+ for (int i = 0; i < block_size; i++) {
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ }
+
+ pre_v = deZigzag(current_delta) + min_delta + pre_v;
+ value_list[value_pos_arr[0]] =pre_v;
+ value_pos_arr[0] ++;
+ }
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ block_size --;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += deZigzag(cur_delta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += deZigzag(currentDelta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ block_size --;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_bos_b";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+// for (int file_i = 8; file_i < 9; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f = tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+ int repeatTime2 = 500;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+
+ }
+ writer.close();
+
+ }
+ }
+
+ @Test
+ public void BOSImproveDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/sprintz_bos_b";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSImproveEncodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_bos_b_improve";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSHTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSHTest.java
deleted file mode 100644
index f14d94e..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSHTest.java
+++ /dev/null
@@ -1,1464 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-
-import static java.lang.Math.*;
-
-public class SPRINTZBOSHTest {
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
- public static int zigzag(int num) {
- if (num < 0) return ((-num)<<1)-1;
- else return num<<1;
- }
-
- public static int deZigzag(int num) {
- if (num % 2 == 0) return num>>1;
- else return -((num+1)>>1);
- }
-
- public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
-
- public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx)<< (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos ++;
- bufIdx++;
- if (bufIdx >= width) {
- return ;
- }
- }
- }
-
- }
-
- public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add ((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size()-start) / 8;
- for(int i=0;i<block_num;i++){
- pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
- encode_pos +=bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values( encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
- }
- return result_list;
- }
-
- // -----------------------------------------------------------------
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- ArrayList<Integer> min_delta,
- int supple_length) {
-
- int[] ts_block_delta = new int[ts_block.length+supple_length-1];
- int block_size = ts_block.length-1;
-
- for (int i = 0; i < block_size; i++) {
-
- int epsilon_v = ts_block[i+1] - ts_block[i];
- epsilon_v = zigzag(epsilon_v);
- ts_block_delta[i] = epsilon_v;
- }
- for(int i = block_size;i<block_size+supple_length;i++){
- ts_block_delta[i] = 0;
- }
- min_delta.add(ts_block[0]);
- return ts_block_delta;
- }
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta) {
- int[] ts_block_delta = new int[remaining-1];
-
- int base = i*block_size+1;
- int end = i*block_size+remaining;
- min_delta[0]=ts_block[base-1];
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- for (int j = base; j < end; j++) {
- int epsilon_v = ts_block[j] - ts_block[j - 1];
- epsilon_v = zigzag(epsilon_v);
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
- ts_block_delta[j-base] =epsilon_v;
-
- }
- for (int j = 0; j < remaining-1; j++) {
- ts_block_delta[j] =ts_block_delta[j]-value_delta_min;
-
- }
- min_delta[1] = (value_delta_min);
- min_delta[2] = (value_delta_max-value_delta_min);
-
-
- return ts_block_delta;
- }
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- return encode_pos;
-
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
-
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if ( cur_value<= final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
- k1++;
-
-
- } else if (cur_value >= final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value-1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
-
- int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
-
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
-
- int2Bytes(k_byte,encode_pos,cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0],encode_pos,cur_byte); // x0
- encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte); // x_min
- encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);//final_left_max
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- }
- encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
- return encode_pos;
-
- }
-
- // sprintz need to substitute ---- start
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining, int encode_pos, byte[] cur_byte) {
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
-
- block_size = remaining - 1;
- int max_delta_value = min_delta[2];
-
- int boundary_index = (int) ((double) block_size / (double) (getBitWith(block_size-1)-1));
-
- int max_bit_width = getBitWith(max_delta_value) + 1;
- int[] lambda_list = {getBitWith(block_size-1),1};
- int lambda_0 = lambda_list[0];
-
- int count_x_min = 0;
- int count_x_max = 0;
- GroupL[] groupL = new GroupL[max_bit_width];
- GroupU[] groupU = new GroupU[max_bit_width];
- for (int i = 0; i < max_bit_width; i++) {
- int[] numbers = new int[block_size];
- int count = 0;
- groupL[i] = new GroupL(numbers, count, i);
- int[] numbersU = new int[block_size];
- groupU[i] = new GroupU(numbersU, count, i);
- }
- for (int value : ts_block_delta) {
- int alpha_i = getBitWith(value);
- if (value == 0) {
- count_x_min++;
- } else {
- groupL[alpha_i].addNumber(value); // (x_min+2^{alpha_i-1},x_min+2^{alpha_i})
- }
- int gamma_i = getBitWith(max_delta_value - value);
- if (value == max_delta_value) {
- count_x_max++;
- } else {
- groupU[gamma_i].addNumber(value);// (x_max-2^{gamma_i},x_max-2^{gamma_i-1})
- }
- }
- int[] alpha_list = new int[max_bit_width];
- int num_of_alpha_list = 0;
- int[] gamma_list = new int[max_bit_width];
- int num_of_gamma_list = 0;
- for (int i = 1; i < max_bit_width; i++) {
- if(groupL[i].count!=0){
- groupL[i].setCount_array();
- alpha_list[num_of_alpha_list] = i;
- num_of_alpha_list ++;
- }
- if(groupU[i].count!=0){
- groupU[i].setCount_array(max_delta_value);
- gamma_list[num_of_gamma_list] = i;
- num_of_gamma_list ++;
- groupU[i].setinvert();
- }
- }
-
-
- int final_k_start_value = -1;//getUniqueValue(sorted_value_list[0], left_shift);
- int final_k_end_value = max_delta_value + 1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
-
- int alpha_size = getBitWith(max(max_delta_value-1,0));
-
- if( num_of_alpha_list ==0 && num_of_gamma_list==0){
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
- return encode_pos;
- }
-
-
- // -------------x_min-------------------
- // lower: x_min
- // upper: x_max
- int k1 = count_x_min;
- int k1_start = 0;
- int k2 = count_x_max;
- int k2_end = max_delta_value;
- int cur_bits = Math.min((k1 + k2) * getBitWith(block_size - 1), block_size + k1 + k2);
- cur_bits += k1;
- cur_bits += k2;
- cur_bits += (block_size - k1 - k2) * getBitWith(k2_end - k1_start - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
- // lower: x_min
- // upper: x_max - 1, x_max - 2, x_max - 4 ...
- //(x_max - 2^{gamma}, x_max - 2^{gamma-1} ]
-
- for (int gamma_i = 0; gamma_i < num_of_gamma_list-1; gamma_i++) {
-
- int gamma = gamma_list[gamma_i];
-
- int gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma-1} gamma : 1 to alpha_size-1
-
- GroupU cur_group_gamma = groupU[gamma];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
- int cur_gamma_count = cur_group_gamma.count;
-
- for(int unique_i=0;unique_i<gamma_unique_number;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;//min_upper_outlier
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- k2 += cur_gamma_count;
-
- }
-
- // lower: x_min
- // k2_end = x_max - 2^{alpha_size-1}
- // upper: (0,x_max - 2^{alpha_size-1}]
-
- int gap_gamma = (int) pow(2,alpha_size-1);
- k2_end = max_delta_value - gap_gamma;
- GroupU cur_group_gamma = groupU[alpha_size];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
-
- for(int unique_i=0;unique_i<gamma_unique_number-1;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * alpha_size;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- // -------------x_min-------------------
-
-
-
-
- // [x_min+2^{alpha-1},x_min+2^{alpha}) alpha = 1 to alphasize - 1
- // -----------------[x_min+2^{alpha-1},x_min+2^{alpha})-------------------
- // alpha_size = getBitWith(max_delta_value);
-
- for (int alpha_i = 0; alpha_i < num_of_alpha_list-1; alpha_i++) {
-
- int alpha = alpha_list[alpha_i];
- int gap_alpha = (int) pow(2,alpha-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha];
-
-
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: x_max
-
- int alpha_value_count = cur_group_alpha.count;
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- // k1; count[x_min, x_min+2^{alpha-1}]
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int unique_i_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
- k2 = count_x_max;
- k2_end = max_delta_value;
- cur_bits = Math.min((unique_i_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_i_cur_k1 + k2);
- cur_bits += unique_i_cur_k1 * alpha;
- cur_bits += k2;
- if (unique_i_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_i_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
-
-
- int pow_2_alpha = (int) pow(2, alpha);
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
-
-
- int cur_k1 = k1; //count[x_min, x_min+2^{alpha-1}]
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- int gamma_size = (int) (log(max_delta_value - (int) pow(2, alpha)) / log(2));
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- int cur_k2 = k2; // count(x_max)
- int gamma_i = 0;
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- if(gamma > gamma_size){
- break;
- }
-
-
- // k1: count[x_min,x_min+2^{alpha-1}]
-
- gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma;
- cur_group_gamma = groupU[gamma]; // (x_max - 2^{gamma}, x_max - 2^{gamma-1})
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
-
- int pow_2_gamma = gap_gamma*2;
-
- // prop 5.7
-
- int cur_k1_start = k1_start + gap_alpha - 1;
- int cur_k2_end = k2_end - gap_gamma + 1;
- int cur_k2_start_of_lambda = cur_k2 + gamma_value_count;// count[x_max - 2^{gamma}+1, x_max]
- int cur_k1_start_of_lambda = k1 + alpha_value_count; // count[x_min, 2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_start_of_lambda + cur_k2_start_of_lambda) * lambda_0, block_size + cur_k1_start_of_lambda + cur_k2_start_of_lambda); // (lambda * (cur_cur_k1 + cur_k2_start_of_lambda) + zeta);
- cur_bits += (cur_k1_start_of_lambda * alpha);
- cur_bits += (cur_k2_start_of_lambda * gamma);
- cur_bits += ((block_size - cur_k1_start_of_lambda - cur_k2_start_of_lambda) * getBitWith(cur_k2_end - cur_k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = cur_k1_start;
- final_k_end_value = cur_k2_end;
- }
-
- // prop 5.4
- int prop_5_4_k1 = cur_k1 + alpha_value_count; //count[x_min, x_min+2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((prop_5_4_k1 + cur_k2) * lambda_0, block_size + prop_5_4_k1 + cur_k2); //(lambda * (prop_5_4_k1 + prop_5_4_k2) + zeta);
- cur_bits += (prop_5_4_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - prop_5_4_k1 - cur_k2) * getBitWith(k2_end - k1_start - gap_alpha - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + gap_alpha - 1;
- final_k_end_value = k2_end;
- }
-
- // prop 5.5
- int prop_5_5_k2 = cur_k2 + gamma_value_count; // count[x_max - 2^{gamma}+1,x_max}
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + prop_5_5_k2) * lambda_0, block_size + cur_k1 + prop_5_5_k2);// (lambda * (prop_5_5_k1 + prop_5_5_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (prop_5_5_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - prop_5_5_k2) * getBitWith(k2_end - gap_gamma - k1_start - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - gap_gamma + 1;
- }
-
- // prop 5.6
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + cur_k2) * lambda_0, block_size + cur_k1 + cur_k2);//(lambda * (prop_5_6_k1 + prop_5_6_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - cur_k2) * getBitWith(k2_end - k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
-
-
- if ((max_pow_2_lambda + 1) * pow_2_alpha + pow_2_gamma <= max_delta_value
- && pow_2_alpha + (max_pow_2_lambda + 1) * pow_2_gamma <= max_delta_value) {
-
-
- }
- else {
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}): k2_end - gap_gamma, k2_end
- // beta: getBitWidth(x_max - 2^{gamma} -(x_min+2^{alpha})) to getBitWidth(x_max - 2^{gamma-1}-(x_min+2^{alpha-1}) )
- int min_beta = getBitWith(max_delta_value-pow_2_gamma-pow_2_alpha);
- int max_beta = getBitWith(k2_end-k1_start);
-
-
-
- for(int beta = min_beta;beta <= max_beta;beta++) {
- int pow_2_beta = (int) pow(2, beta);
- if (beta >= alpha + max_lambda && beta <= gamma + 1 ){
- if(pow_2_beta + pow_2_alpha + gap_gamma >= max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- }else if (beta <= alpha + 1 && beta >= gamma + max_lambda) {
- if(pow_2_beta + gap_alpha + pow_2_gamma >= max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- } else if (beta <= alpha + 1 && beta <= gamma + 1) {
- if(pow_2_beta + gap_alpha + gap_gamma > max_delta_value){
-
- }else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
- }else{
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.min(Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2)),gap_gamma-1);
- int x_u_i_end = Math.max(0,Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma));
- int unique_s = cur_group_gamma.invert[x_u_i_start];
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
-
- for (int unique_i = unique_s; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
-
- }
-
- }
-
- cur_k2 = cur_k2 + gamma_value_count; // += count(x_max - 2^{gamma}, x_max - 2^{gamma-1})
-
- if(cur_k1+cur_k2>boundary_index){
- max_pow_2_lambda = (int) pow(2, 1);
- max_lambda = 1;
- }
-
- }
-
-
- k2 = cur_k2;
-
- // gamma: gamma_size+1 to ....
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- //k2_end = x_max - 2^{gamma-1} ; gamma : 1 to gamma_size
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]
-
-
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- cur_group_gamma = groupU[gamma];
- k2_end = max_delta_value - gap_gamma;
-
- if(k2_end <= k1_start){
- break;
- }
- int gamma_value_count = cur_group_gamma.count;
-
- gamma_unique_number = cur_group_gamma.unique_number;
- gamma_sorted = cur_group_gamma.sorted_value_list;
-
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int unique_j_cur_k1 = cur_k1+ cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- int x_u_i_end = Math.max(0,Math.min(k2_end - k1_start - x_l_i,gap_gamma));
- int unique_e = cur_group_gamma.invert[x_u_i_end];
- for (int unique_i = 0; unique_i < unique_e; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- int unique_i_cur_k2 = cur_k2+ cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- }
- cur_k2 += gamma_value_count;
- }
-
- k1 += alpha_value_count;
- }
- // -------------------[x_min+2^{alpha-1},x_min+2^{alpha})-----------------
-
-
- // [x_min+2^{alpha_size-1},x_max)
- // ----------------[x_min+2^{alpha_size-1},x_max)--------------------
-
- // x_min+2^{alpha_size-1} < x_max && x_min+2^{alpha_size} >= x_max
- // alpha_size - 1 < log(xmax) && alpha_size >= log(xmax)
- // alpha_size = \lceil log(x_max-1+1) \rceil
-
- alpha_size = alpha_list[num_of_alpha_list-1];
- int gap_alpha = (int) pow(2,alpha_size-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha_size];
- int alpha_value_count = cur_group_alpha.count;
-
-
- // lower: [x_min+2^{alpha_size-1},x_max)
- // ----------------- [x_min+2^{alpha_size-1},x_max)----------------------
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- int x_l_i_end = max_delta_value - k1_start; // [1, x_max - x_min+2^{alpha_size-1})
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- if (x_l_i >= x_l_i_end) {
- break;
- }
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
-
- // lower: [x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: x_max
- k2 = count_x_max;
- k2_end = max_delta_value;
-
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + k2);
- cur_bits += unique_j_cur_k1 * alpha_size;
- cur_bits += k2;
- if (unique_j_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
- // lower: (x_min+2^{alpha_size-1},x_max)
- // upper: (x_max - 2^{gamma}, x_max)
- // gamma: gamma_size+1 to ....
- // lower: (x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}] : k2_end - gap_gamma, k2_end
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- for (int gamma_i = 0; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma} ;
-
- if(k2_end<=k1_start){
- break;
- }
- cur_group_gamma = groupU[gamma];
-
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
- int pow_2_gamma = gap_gamma*2;
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- int x_u_i_end = k2_end - k1_start - x_l_i; // x_u_i>= k2_end - k1_start - x_l_i; break ;
-
- if(x_u_i_end <= 0 || x_l_i >= x_l_i_end){
- break;
- }
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- x_u_i_end = Math.min(x_u_i_end,gap_gamma - 1);
- int unique_e = cur_group_gamma.invert[x_u_i_end];
-
-
- for(int unique_i=0;unique_i<unique_e;unique_i++){
-
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha_size;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- }
-
-
- k2 += gamma_value_count;
-
-
-
- }
- // ------------------ [x_min+2^{alpha_size-1},x_max)--------------------
-
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
-
- return encode_pos;
- }
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
- block_size++;
-
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all,encode_pos,encoded_result);
- encode_pos += 4;
- int block_num = length_all / block_size;
- int2Bytes(block_size,encode_pos,encoded_result);
- encode_pos+= 4;
-
- for (int i = 0; i < block_num; i++) {
-
- encode_pos = BOSBlockEncoder(data, i, block_size,block_size, encode_pos,encoded_result);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- } else {
-
- int start = block_num * block_size;
- int remaining = length_all-start;
-
-
- encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
-
-
- }
-
-
- return encode_pos;
- }
-
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
- int value0 = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] =value0;
- value_pos_arr[0] ++;
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
-
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
-
- for (int i = 0; i < block_size; i++) {
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- }
-
- pre_v = deZigzag(current_delta) + min_delta + pre_v;
- value_list[value_pos_arr[0]] =pre_v;
- value_pos_arr[0] ++;
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
- int[] value_list = new int[length_all+block_size];
- block_size --;
-
- int[] value_pos_arr = new int[1];
- for (int k = 0; k < block_num; k++) {
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
-
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- remain_length --;
- BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
- }
- }
-
- public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
- String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/sprintz_pruning";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
- int repeatTime2 = 100;
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
-
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
-// f = tempList[1];
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
- loader.readHeaders();
-
- while (loader.readRecord()) {
-
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
-
- }
-
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
- String[] record = {
- f.toString(),
- "SPRINTZ+BOS-P",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
- }
- writer.close();
-
- }
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSMTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSMTest.java
new file mode 100644
index 0000000..7e053c1
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSMTest.java
@@ -0,0 +1,2301 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
+import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+
+import static java.lang.Math.*;
+import static org.apache.iotdb.tsfile.constant.TestConstant.random;
+
+public class SPRINTZBOSMTest {
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int findMedianImprove(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ return quickSelect2(arr, 0, n - 1, n / 2);
+ }
+
+ private static int quickSelect2(int[] arr, int left, int right, int k){
+ int pivotV=arr[left+random.nextInt(right-left+1)],tmpV;
+ int posEqual=left,posSmaller=left; // a[left,posEqual): = pivotV; a[posEqual,posSmaller): < pivotV
+ for(int i=left;i<=right;i++){
+ if(arr[i]==pivotV){
+ tmpV=arr[i];
+ arr[i]=arr[posSmaller];
+ arr[posSmaller]=arr[posEqual];
+ arr[posEqual]=tmpV;
+ posEqual++;
+ posSmaller++;
+ }else
+ if(arr[i]<pivotV){
+ tmpV=arr[posSmaller];
+ arr[posSmaller]=arr[i];
+ arr[i]=tmpV;
+ posSmaller++;
+ }
+ }
+ if(k+(posEqual-left)<=posSmaller-1)return quickSelect2(arr,posEqual,posSmaller-1,k+(posEqual-left));
+ else if(k<=posSmaller-1)return pivotV;
+ else return quickSelect2(arr,posSmaller,right,k);
+ }
+ public static int findMedian(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ return quickSelect(arr, 0, n - 1, n / 2);
+ }
+
+ private static int quickSelect(int[] arr, int left, int right, int k) {
+ if (left == right) {
+ return arr[left];
+ }
+
+ int pivotIndex = partition(arr, left, right);
+ if (k == pivotIndex) {
+ return arr[k];
+ } else if (k < pivotIndex) {
+ return quickSelect(arr, left, pivotIndex - 1, k);
+ } else {
+ return quickSelect(arr, pivotIndex + 1, right, k);
+ }
+ }
+
+ private static int partition(int[] arr, int left, int right) {
+ int pivot = arr[right];
+ int i = left;
+ for (int j = left; j < right; j++) {
+ if (arr[j] <= pivot) {
+ swap(arr, i, j);
+ i++;
+ }
+ }
+ swap(arr, i, right);
+ return i;
+ }
+
+ private static void swap(int[] arr, int i, int j) {
+ int temp = arr[i];
+ arr[i] = arr[j];
+ arr[j] = temp;
+ }
+ public static int zigzag(int num) {
+ if (num < 0) return ((-num)<<1)-1;
+ else return num<<1;
+ }
+
+ public static int deZigzag(int num) {
+ if (num % 2 == 0) return num>>1;
+ else return -((num+1)>>1);
+ }
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+ }
+ return result_list;
+ }
+
+ // -----------------------------------------------------------------
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining-1];
+
+ int base = i*block_size+1;
+ int end = i*block_size+remaining;
+ min_delta[0]=ts_block[base-1];
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ for (int j = base; j < end; j++) {
+ int epsilon_v = ts_block[j] - ts_block[j - 1];
+ epsilon_v = zigzag(epsilon_v);
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ ts_block_delta[j-base] =epsilon_v;
+
+ }
+ for (int j = 0; j < remaining-1; j++) {
+ ts_block_delta[j] =ts_block_delta[j]-value_delta_min;
+
+ }
+ min_delta[1] = (value_delta_min);
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+
+ return ts_block_delta;
+ }
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width));
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static void addToArchiveCompression(SevenZOutputFile out, File file, String dir) {
+ String name = dir + File.separator + file.getName();
+ if(dir.equals(".")) {
+ name = file.getName();
+ }
+ if (file.isFile()){
+ SevenZArchiveEntry entry = null;
+ FileInputStream in = null;
+ try {
+ entry = out.createArchiveEntry(file, name);
+ out.putArchiveEntry(entry);
+ in = new FileInputStream(file);
+ byte[] b = new byte[1024];
+ int count = 0;
+ while ((count = in.read(b)) > 0) {
+ out.write(b, 0, count);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ out.closeArchiveEntry();
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ } else if (file.isDirectory()) {
+ File[] children = file.listFiles();
+ if (children != null){
+ for (File child : children){
+ addToArchiveCompression(out, child, name);
+ }
+ }
+ } else {
+ System.out.println(file.getName() + " is not supported");
+ }
+ }
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedianImprove(findMedianArray);
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value<=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+
+
+ }
+
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = (int) pow(2,beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size,block_size, encode_pos,encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+// int supple_length;
+// if (remaining_length % 8 == 0) {
+// supple_length = 1;
+// } else if (remaining_length % 8 == 1) {
+// supple_length = 0;
+// } else {
+// supple_length = 9 - remaining_length % 8;
+// }
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, supple_length, encode_pos,encoded_result);
+ }
+
+
+ return encode_pos;
+ }
+
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_k_start_value-1);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+
+ if(k1==0 && k2==0){
+ bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
+ }
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedian(findMedianArray);
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value<=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+
+
+ }
+
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = (int) pow(2,beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+
+ encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoder(data, i, block_size,block_size, encode_pos,encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+// int supple_length;
+// if (remaining_length % 8 == 0) {
+// supple_length = 1;
+// } else if (remaining_length % 8 == 1) {
+// supple_length = 0;
+// } else {
+// supple_length = 9 - remaining_length % 8;
+// }
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, supple_length, encode_pos,encoded_result);
+ }
+
+
+ return encode_pos;
+ }
+
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+ int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ if(k1!=0 || k2 != 0){
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+ }else {
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ }
+
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+
+ for (int i = 0; i < block_size; i++) {
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = final_normal.get(normal_i) + final_k_start_value+1;
+ normal_i++;
+ }
+ }
+
+ pre_v = deZigzag(current_delta) + min_delta + pre_v;
+ value_list[value_pos_arr[0]] =pre_v;
+ value_pos_arr[0] ++;
+ }
+ return decode_pos;
+ }
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += deZigzag(cur_delta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += deZigzag(currentDelta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ block_size --;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ block_size --;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+// for (int file_i = 8; file_i < 9; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f = tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+ int repeatTime2 = 100;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+
+ }
+ writer.close();
+
+ }
+ }
+
+ @Test
+ public void compressTest() throws IOException {
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/sprintz_m_c";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 1;
+// for (int file_i = 8; file_i < 9; file_i++) {
+ CompressionType[] compressList = {
+ CompressionType.LZ4,
+ CompressionType.LZMA2,
+ };
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+
+ for (CompressionType comp : compressList) {
+ double ratio = 0;
+ double compressed_size = 0;
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ byte[] compressed = compressor.compress(encoded_result);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ double ratio = 0;
+ double compressed_size = 0;
+ File outfile = new File(parent_dir + "icde0802/example.bin");
+
+ // 使用FileOutputStream将byte数组写入文件
+ try (FileOutputStream fos = new FileOutputStream(outfile)) {
+ fos.write(encoded_result);
+ } catch (IOException e2) {
+ // 处理可能的I/O异常
+ e2.printStackTrace();
+ }
+
+ File input = new File(parent_dir + "icde0802/example.bin");
+ File output = new File(parent_dir + "icde0802/example.7z");
+ SevenZOutputFile out = new SevenZOutputFile(output);
+
+ addToArchiveCompression(out, input, ".");
+ out.closeArchiveEntry();
+
+ long compressed = output.length();
+
+
+ // test compression ratio and compressed size
+ compressed_size += compressed;
+ double ratioTmp =
+ (double) compressed / (double) (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ "7ZIP",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void ExpTest() throws IOException {// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_exp_test";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("Synthetic_Exp_100");
+ dataset_name.add("Synthetic_Exp_1000");
+ dataset_name.add("Synthetic_Exp_10000");
+ dataset_name.add("Synthetic_Exp_100000");
+ dataset_name.add("Synthetic_Exp_1000000");
+ dataset_name.add("Synthetic_Normal_100");
+ dataset_name.add("Synthetic_Normal_1000");
+ dataset_name.add("Synthetic_Normal_10000");
+ dataset_name.add("Synthetic_Normal_100000");
+ dataset_name.add("Synthetic_Normal_1000000");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/Exp_100.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Exp_1000.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Exp_10000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_100000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_1000000.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_100.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_1000.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Normal_10000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_100000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_1000000.csv");//5
+// dataset_block_size.add(2048);
+
+// for (int file_i = 8; file_i < 9; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f = tempList[1];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+
+ }
+
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+ int repeatTime2 = 1;
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+
+ }
+ writer.close();
+
+ }
+ }
+
+ @Test
+ public void BOSImproveDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/sprintz_bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSImproveEncodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_bos_m_improve";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSVTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSVTest.java
index acf736e..da8590d 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSVTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSVTest.java
@@ -2,6 +2,7 @@
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
@@ -11,7 +12,7 @@
import java.util.ArrayList;
import java.util.Arrays;
-import static java.lang.Math.pow;
+import static java.lang.Math.*;
public class SPRINTZBOSVTest {
@@ -180,6 +181,7 @@
// -----------------------------------------------------------------
+
public static int[] getAbsDeltaTsBlock(
int[] ts_block,
int i,
@@ -215,7 +217,6 @@
return ts_block_delta;
}
-
public static int encodeOutlier2Bytes(
ArrayList<Integer> ts_block_delta,
int bit_width,
@@ -304,7 +305,9 @@
private static int BOSEncodeBits(int[] ts_block_delta,
int final_k_start_value,
+ int final_x_l_plus,
int final_k_end_value,
+ int final_x_u_minus,
int max_delta_value,
int[] min_delta,
int encode_pos,
@@ -385,32 +388,49 @@
int2Bytes(k_byte,encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(min_delta[0],encode_pos,cur_byte); // x0
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte); // x_min
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
int left_bit_width = getBitWith(final_k_start_value);//final_left_max
int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
- for (int i : bitmap_outlier) {
+ if(k1==0 && k2==0){
+ bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
}
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
if (k1 != 0)
encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
@@ -429,6 +449,7 @@
block_size = remaining-1;
int max_delta_value = min_delta[2];
+
int[] value_list = new int[block_size];
int unique_value_count = 0;
int[] value_count_list = new int[max_delta_value+1];
@@ -459,34 +480,81 @@
}
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value ) * (block_size));
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
- int start_value_size = unique_value_count;
- for (int start_value_i = 0; start_value_i < start_value_size; start_value_i++) {
- int k_start_value = getUniqueValue(sorted_value_list[start_value_i], left_shift) ;
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
- int cur_k1 = getCount(sorted_value_list[start_value_i],mask);
+ int k_start_value = -1; // x_l_minus
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
int k_end_value;
int cur_bits;
int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
cur_bits = 0;
cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
- if (cur_k1 != 0)
- cur_bits += cur_k1 * getBitWith(k_start_value);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
if (cur_k1 + cur_k2 != block_size)
- cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(k_end_value - k_start_value-2);
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
if (cur_k2 != 0)
cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
@@ -494,14 +562,16 @@
if (cur_bits < min_bits) {
min_bits = cur_bits;
final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
}
}
}
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
+ encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
return encode_pos;
}
@@ -561,8 +631,438 @@
return encode_pos;
}
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+
+ cur_bits = 0;
+ cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size,block_size, encode_pos,encoded_result);
+
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ } else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+
+// int[] ts_block = new int[length_all-start];
+// if (length_all - start >= 0) System.arraycopy(data, start, ts_block, 0, length_all - start);
+//
+// int supple_length;
+// if (remaining_length % 8 == 0) {
+// supple_length = 1;
+// } else if (remaining_length % 8 == 1) {
+// supple_length = 0;
+// } else {
+// supple_length = 9 - remaining_length % 8;
+// }
+//
+//
+// encode_pos = BOSBlockEncoder(ts_block, supple_length, encode_pos,encoded_result);
+ }
+
+
+ return encode_pos;
+ }
public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
int k_byte = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
@@ -579,81 +1079,92 @@
int min_delta = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
ArrayList<Integer> final_left_outlier = new ArrayList<>();
ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal;
+ ArrayList<Integer> final_normal= new ArrayList<>();;
ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+ int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ if(k1!=0 || k2 != 0){
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
}
- }
- if (remaining_bits == 0) {
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
remaining_bits = 8;
if (bitmap_outlier_i >= bitmap_bytes) break;
tmp = bitmap_outlier.get(bitmap_outlier_i);
bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
}
}
+ i++;
}
- i++;
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
}
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
+ }else {
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
}
@@ -675,7 +1186,7 @@
int right_outlier_i = 0;
int normal_i = 0;
int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
for (int i = 0; i < block_size; i++) {
@@ -748,10 +1259,263 @@
}
}
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += deZigzag(cur_delta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += deZigzag(currentDelta);
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+ int[] value_list = new int[length_all+block_size];
+ block_size --;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/sprintz_bos";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz_bos";
String input_parent_dir = parent_dir + "trans_data/";
ArrayList<String> input_path_list = new ArrayList<>();
ArrayList<String> output_path_list = new ArrayList<>();
@@ -877,7 +1641,7 @@
String[] record = {
f.toString(),
- "SPRINTZ+BOS-O",
+ "SPRINTZ+BOS-V",
String.valueOf(encodeTime),
String.valueOf(decodeTime),
String.valueOf(data1.size()),
@@ -894,4 +1658,150 @@
}
}
+
+ @Test
+ public void BOSImproveTimeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/sprintz_bos_v";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "SPRINTZ+BOS-V",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
similarity index 64%
rename from iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSLTest.java
rename to iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
index 5937c72..ecd724a 100644
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBOSLTest.java
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/SPRINTZBPTest.java
@@ -2,6 +2,7 @@
import com.csvreader.CsvReader;
import com.csvreader.CsvWriter;
+import org.junit.Test;
import java.io.File;
import java.io.IOException;
@@ -12,7 +13,7 @@
import static java.lang.Math.pow;
-public class SPRINTZBOSLTest {
+public class SPRINTZBPTest {
public static int getBitWith(int num) {
if (num == 0) return 1;
@@ -24,7 +25,47 @@
public static int getUniqueValue(long long1, int left_shift) {
return ((int) ((long1) >> left_shift));
}
+ public static int findMedian(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ return quickSelect(arr, 0, n - 1, n / 2);
+ }
+ private static int quickSelect(int[] arr, int left, int right, int k) {
+ if (left == right) {
+ return arr[left];
+ }
+
+ int pivotIndex = partition(arr, left, right);
+ if (k == pivotIndex) {
+ return arr[k];
+ } else if (k < pivotIndex) {
+ return quickSelect(arr, left, pivotIndex - 1, k);
+ } else {
+ return quickSelect(arr, pivotIndex + 1, right, k);
+ }
+ }
+
+ private static int partition(int[] arr, int left, int right) {
+ int pivot = arr[right];
+ int i = left;
+ for (int j = left; j < right; j++) {
+ if (arr[j] <= pivot) {
+ swap(arr, i, j);
+ i++;
+ }
+ }
+ swap(arr, i, right);
+ return i;
+ }
+
+ private static void swap(int[] arr, int i, int j) {
+ int temp = arr[i];
+ arr[i] = arr[j];
+ arr[j] = temp;
+ }
public static int zigzag(int num) {
if (num < 0) return ((-num)<<1)-1;
else return num<<1;
@@ -178,26 +219,7 @@
}
// -----------------------------------------------------------------
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- ArrayList<Integer> min_delta,
- int supple_length) {
- int[] ts_block_delta = new int[ts_block.length+supple_length-1];
- int block_size = ts_block.length-1;
-
- for (int i = 0; i < block_size; i++) {
-
- int epsilon_v = ts_block[i+1] - ts_block[i];
- epsilon_v = zigzag(epsilon_v);
- ts_block_delta[i] = epsilon_v;
- }
- for(int i = block_size;i<block_size+supple_length;i++){
- ts_block_delta[i] = 0;
- }
- min_delta.add(ts_block[0]);
- return ts_block_delta;
- }
public static int[] getAbsDeltaTsBlock(
int[] ts_block,
int i,
@@ -309,7 +331,7 @@
cur_number = int_remaining.get(cur_number_i);
cur_number_i++;
- tmp += (int) (cur_number >> (32 - remain_bits));
+ tmp += (cur_number >> (32 - remain_bits));
result_list.add(tmp);
cur_number <<= remain_bits;
cur_number &= 0xFFFFFFFFL;
@@ -322,7 +344,9 @@
private static int BOSEncodeBits(int[] ts_block_delta,
int final_k_start_value,
+ int final_x_l_plus,
int final_k_end_value,
+ int final_x_u_minus,
int max_delta_value,
int[] min_delta,
int encode_pos,
@@ -403,32 +427,49 @@
int2Bytes(k_byte,encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(min_delta[0],encode_pos,cur_byte); // x0
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte); // x_min
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
int left_bit_width = getBitWith(final_k_start_value);//final_left_max
int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
- for (int i : bitmap_outlier) {
+ if(k1==0 && k2==0){
+ bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
}
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
if (k1 != 0)
encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
@@ -438,7 +479,7 @@
}
- // sprintz need to substitute ---- start
+
private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
int[] min_delta = new int[3];
@@ -446,140 +487,126 @@
block_size = remaining-1;
-
-
-
int max_delta_value = min_delta[2];
- int[] alpha_count_list = new int[getBitWith(max_delta_value)+1];//count(xmin) count(xmin + 2) count(xmin + 4)... count(xmax)
- int[] alpha_box_count_list = new int[getBitWith(max_delta_value)];// count(xmin, xmin + 2), count(xmin + 2, xmin + 4)...
- int[] gamma_count_list = new int[getBitWith(max_delta_value)+1];
- int[] gamma_box_count_list = new int[getBitWith(max_delta_value)];
- for(int value:ts_block_delta){
- if (value == 0){
- alpha_count_list[0]++;
- }
- else if (value == pow(2,getBitWith(value)-1) && value != 1){
- alpha_count_list[getBitWith(value)-1]++;
- }else {
- alpha_box_count_list[getBitWith(value)-1]++;
- }
- if (value == max_delta_value){
- gamma_count_list[0]++;
- }
- else if (max_delta_value - value == pow(2,getBitWith(max_delta_value-value)-1) && max_delta_value - value != 1){
- gamma_count_list[getBitWith(max_delta_value-value)-1]++;
- }else {
- gamma_box_count_list[getBitWith(max_delta_value-value)-1]++;
- }
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+ int bit_width_final = getBitWith(max_delta_value);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ for(int i : ts_block_delta){
+ final_normal.add(i);
}
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value ) * (block_size));
-
- int alpha_size = getBitWith(max_delta_value);
- int cur_k1_close = 0;
- for (int alpha = 1; alpha < alpha_size; alpha++) {
- int k_start_value_close = (int) pow(2,alpha-1);
- cur_k1_close += alpha_count_list[alpha-1];
- if (alpha > 1) {
- cur_k1_close += alpha_box_count_list[alpha - 2];
- }
- int cur_k2_close = 0;
- int k_end_value_close;
- int cur_bits;
-
- for (int gamma = 1; (int) pow(2,gamma) + (int) pow(2,alpha) <= max_delta_value + 1; gamma++) {
-
- k_end_value_close = max_delta_value - (int) pow(2,gamma-1);
-
- cur_bits = 0;
- cur_k2_close += gamma_count_list[gamma-1];
- if (gamma > 1){
- cur_k2_close += gamma_box_count_list[gamma-2];
- }
-
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * alpha;
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close-2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- //C2
- cur_bits = 0;
- int cur_k1_open = cur_k1_close - alpha_count_list[alpha - 1];
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma - 1];
- int k_start_value_open = k_start_value_close - 1;
- int k_end_value_open = k_end_value_close + 1;
- cur_bits += Math.min((cur_k1_open + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_open);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha - 1);
- if (cur_k1_open + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_open - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * (gamma - 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_open;
- }
-
- //C3 k1 open k2 close
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha - 1);
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- }
-
- //C4 k1 close k2 open
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * alpha;
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * (gamma - 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
-
-
- }
- }
-
- // RLE need to substitute ----- end
-
-
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
return encode_pos;
}
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+ int bit_width_final = getBitWith(min_delta[2]);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+// ArrayList<Integer> final_normal = new ArrayList<>();
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+ for(int value:ts_block_delta){
+ encode_pos = EncodeBits(value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ }
+ else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
public static int BOSEncoder(
int[] data, int block_size, byte[] encoded_result) {
block_size++;
@@ -638,14 +665,11 @@
public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
- int k2 = (int) (k_byte / pow(2, 16));
+ ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int bit_width_final = 0;
int value0 = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
value_list[value_pos_arr[0]] =value0;
@@ -654,135 +678,21 @@
int min_delta = bytes2Integer(encoded, decode_pos, 4);
decode_pos += 4;
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
-
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
- }
-
-
ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size , decode_pos_normal);
decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
+
int normal_i = 0;
int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
for (int i = 0; i < block_size; i++) {
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = final_normal.get(normal_i) + final_k_start_value+1;
- normal_i++;
- }
- }
-
- pre_v = deZigzag(current_delta) + min_delta + pre_v;
+ int current_delta= final_normal.get(normal_i);
+ pre_v = deZigzag(current_delta + min_delta) + pre_v;
value_list[value_pos_arr[0]] =pre_v;
value_pos_arr[0] ++;
}
@@ -804,7 +714,10 @@
block_size --;
int[] value_pos_arr = new int[1];
+// System.out.println(length_all);
+// System.out.println(encoded.length);
for (int k = 0; k < block_num; k++) {
+// System.out.println(k);
decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
}
@@ -822,10 +735,127 @@
}
}
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += deZigzag(cur_delta);
+ value_list[value_pos_arr[0]++] = pre_v;
+ }
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// value_pos_arr[0] = valuePos;
+// return decode_list[0];
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// ArrayList<Integer> final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// int normal_i = 0;
+//// int pre_v = value0;
+//
+// for (int i = 0; i < block_size; i++) {
+// int current_delta = min_delta + final_normal.get(normal_i) ;
+// pre_v = current_delta + pre_v;
+// value_list[value_pos_arr[0]] = pre_v;
+// value_pos_arr[0]++;
+// }
+//
+// return decode_pos;
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
public static void main(@org.jetbrains.annotations.NotNull String[] args) throws IOException {
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/";// your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/sprintz_bws";
+ int repeatTime2 = 100;
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/sprintz";
String input_parent_dir = parent_dir + "trans_data/";
ArrayList<String> input_path_list = new ArrayList<>();
ArrayList<String> output_path_list = new ArrayList<>();
@@ -874,7 +904,7 @@
output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
// dataset_block_size.add(1024);
-// for (int file_i = 8; file_i < 9; file_i++) {
+// for (int file_i = 7; file_i < 8; file_i++) {
for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
String inputPath = input_path_list.get(file_i);
@@ -902,7 +932,7 @@
assert tempList != null;
for (File f : tempList) {
-
+// f = tempList[1];
System.out.println(f);
InputStream inputStream = Files.newInputStream(f.toPath());
@@ -929,7 +959,7 @@
long decodeTime = 0;
double ratio = 0;
double compressed_size = 0;
- int repeatTime2 = 100;
+
int length = 0;
@@ -949,17 +979,17 @@
e = System.nanoTime();
decodeTime += ((e - s) / repeatTime2);
- String[] record = {
- f.toString(),
- "SPRINTZ+BWS",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
+ String[] record = {
+ f.toString(),
+ "SPRINTZ",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
}
@@ -968,4 +998,5 @@
}
}
+
}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSBImproveTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSBImproveTest.java
new file mode 100644
index 0000000..cecedce
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSBImproveTest.java
@@ -0,0 +1,2389 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static java.lang.Math.min;
+import static java.lang.Math.pow;
+
+public class TSDIFFBOSBImproveTest {
+
+ public static long combine2Int(int int1, int int2) {
+ return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
+ }
+
+ public static int getTime(long long1) {
+ return ((int) (long1 >> 32));
+ }
+
+ public static int getValue(long long1) {
+ return ((int) (long1));
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+// return encode_pos;
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+
+ }
+ return result_list;
+ }
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining-1];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i*block_size+1;
+ int end = i*block_size+remaining;
+
+ int tmp_j_1 = ts_block[base-1];
+ min_delta[0] =tmp_j_1;
+ int j = base;
+ int tmp_j;
+
+ while(j<end){
+ tmp_j = ts_block[j];
+ int epsilon_v = tmp_j - tmp_j_1;
+ ts_block_delta[j-base] = epsilon_v;
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ tmp_j_1 = tmp_j;
+ j++;
+ }
+ j = 0;
+ end = remaining -1;
+ while(j<end){
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+ j++;
+ }
+
+ min_delta[1] = value_delta_min;
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+
+ return ts_block_delta;
+ }
+
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (int) (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_x_l_plus);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+
+ if(k1==0 && k2==0){
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
+ }
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
+
+// if(k1+k2!=block_size)
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+// int beta_max_all = getBitWith(max_delta_value)+1;
+// int[][] hash_table_count = new int[unique_value_count][beta_max_all];
+// int[][] hash_table_value = new int[unique_value_count][beta_max_all];
+// int cur_value = getUniqueValue(sorted_value_list[0], left_shift) ;
+// int next_value = getUniqueValue(sorted_value_list[1], left_shift) ;
+//
+// for (int value_i = 0; value_i < unique_value_count; value_i++) {
+//
+//
+// next_value = getUniqueValue(sorted_value_list[value_i + 1], left_shift) ;
+// long k_start_valueL = sorted_value_list[value_i];
+// hash_table_count[value_i][0] = getCount(k_start_valueL,mask);
+//
+// int beta_max = getBitWith(max_delta_value - cur_value);
+// for(int beta = 1; beta <= beta_max; beta++){
+//
+// }
+// cur_value = next_value ;
+//
+// }
+
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = max_delta_value - (1<<gamma) + 1;
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2, gamma) + 1);
+//
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+
+
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 0; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value < x_u_plus_pow_beta && k_end_value < x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+//
+
+ }
+
+ for(int beta = 0; beta < gamma_max; beta++){
+
+ int pow_beta = 1<<beta;
+ int start_value_i = 0;
+ int end_value_i = start_value_i+1;
+
+ for (; start_value_i < unique_value_count-1; start_value_i++) {
+ long x_l_minusL = sorted_value_list[start_value_i];
+ int x_l_minus = getUniqueValue(x_l_minusL, left_shift) ;
+ int x_l_plus = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+ int x_u_plus_pow_beta = pow_beta+x_l_plus;
+ if(x_u_plus_pow_beta > max_delta_value) break;
+
+
+
+ cur_k1 = getCount(x_l_minusL,mask);
+ int lower_outlier_cost = cur_k1 * getBitWith(x_l_minus);
+
+ while ( end_value_i < unique_value_count) {
+// if(beta==3 && end_value_i==22)
+// {
+// System.out.println(x_l_minus);
+// System.out.println(x_l_plus);
+// }
+
+ int x_u_minus = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus < x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta){
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus - x_l_plus);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = x_l_minus;
+ final_x_l_plus = x_l_plus;
+ final_k_end_value = x_u_plus;
+ final_x_u_minus = x_u_minus;
+ }
+ break;
+ }
+// else if (x_u_minus >= x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta) {
+// break;
+// }
+
+ end_value_i++;
+ }
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+// System.out.println(encode_pos);
+
+ return encode_pos;
+ }
+
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ }
+ else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+ int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ if(k1!=0 || k2 != 0){
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+ }else {
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ }
+
+
+
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+
+ for (int i = 0; i < block_size; i++) {
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = min_delta + final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ }
+ }
+
+ pre_v = current_delta + pre_v;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+// int beta_max_all = getBitWith(max_delta_value)+1;
+// int[][] hash_table_count = new int[unique_value_count][beta_max_all];
+// int[][] hash_table_value = new int[unique_value_count][beta_max_all];
+// int cur_value = getUniqueValue(sorted_value_list[0], left_shift) ;
+// int next_value = getUniqueValue(sorted_value_list[1], left_shift) ;
+//
+// for (int value_i = 0; value_i < unique_value_count; value_i++) {
+//
+//
+// next_value = getUniqueValue(sorted_value_list[value_i + 1], left_shift) ;
+// long k_start_valueL = sorted_value_list[value_i];
+// hash_table_count[value_i][0] = getCount(k_start_valueL,mask);
+//
+// int beta_max = getBitWith(max_delta_value - cur_value);
+// for(int beta = 1; beta <= beta_max; beta++){
+//
+// }
+// cur_value = next_value ;
+//
+// }
+
+
+ int gamma_max = getBitWith(max_delta_value);
+ int[] gamma_count_list = new int[gamma_max+1];
+ int[] x_u_minus_value_list = new int[gamma_max+1];
+ int[] x_u_plus_value_list = new int[gamma_max+1];
+ int end_i = unique_value_count - 1;
+ for(int gamma = 0; gamma <= gamma_max; gamma++) {
+ int x_u_plus_pow_beta = max_delta_value - (1<<gamma) + 1;
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2, gamma) + 1);
+//
+ for (; end_i > 0; end_i--) {
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_i - 1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_i], left_shift);
+ if (x_u_minus_value < x_u_plus_pow_beta && x_u_plus_value >= x_u_plus_pow_beta){
+ gamma_count_list[gamma] = getCount(sorted_value_list[end_i-1],mask);
+ x_u_minus_value_list[gamma] = x_u_minus_value;
+ x_u_plus_value_list[gamma] = x_u_plus_value;
+ } else if (x_u_minus_value < x_u_plus_pow_beta) {
+ break;
+ }
+ }
+ }
+ for(int gamma = 1; gamma < gamma_max; gamma++) {
+ if(gamma_count_list[gamma]==0){
+ gamma_count_list[gamma] = gamma_count_list[gamma-1];
+ x_u_minus_value_list[gamma] = x_u_minus_value_list[gamma-1];
+ x_u_plus_value_list[gamma] = x_u_plus_value_list[gamma-1];
+ }
+ }
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ int beta_max = getBitWith(max_delta_value - x_l_plus_value);
+
+ int lower_outlier_cost = cur_k1 * getBitWith(k_start_value);
+
+
+
+ for(int gamma = 0; gamma < beta_max; gamma++){
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+ x_u_minus_value = x_u_minus_value_list[gamma];
+ k_end_value = x_u_plus_value_list[gamma];
+ cur_bits = 0;
+ cur_k2 = block_size - gamma_count_list[gamma];
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+// end_value_i = unique_value_count - 1;
+// for(int gamma = 0; gamma <= beta_max; gamma++){
+// for (; end_value_i > start_value_i; end_value_i--) {
+// int x_u_plus_pow_beta = (int) (max_delta_value - pow(2,gamma)+1);
+// x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+// k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+// if(x_u_minus_value < x_u_plus_pow_beta && k_end_value >= x_u_plus_pow_beta){
+// cur_bits = 0;
+// cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+//
+// cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+// cur_bits += cur_k1 * getBitWith(k_start_value);
+// if (cur_k1 + cur_k2 != block_size)
+// cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+// if (cur_k2 != 0)
+// cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+//
+//
+// if (cur_bits < min_bits) {
+// min_bits = cur_bits;
+// final_k_start_value = k_start_value;
+// final_x_l_plus = x_l_plus_value;
+// final_k_end_value = k_end_value;
+// final_x_u_minus = x_u_minus_value;
+// }
+// } else if (x_u_minus_value < x_u_plus_pow_beta && k_end_value < x_u_plus_pow_beta) {
+// break;
+// }
+// }
+// }
+//
+
+ }
+
+ for(int beta = 0; beta < gamma_max; beta++){
+
+ int pow_beta = 1<<beta;
+ int start_value_i = 0;
+ int end_value_i = start_value_i+1;
+
+ for (; start_value_i < unique_value_count-1; start_value_i++) {
+ long x_l_minusL = sorted_value_list[start_value_i];
+ int x_l_minus = getUniqueValue(x_l_minusL, left_shift) ;
+ int x_l_plus = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+ int x_u_plus_pow_beta = pow_beta+x_l_plus;
+ if(x_u_plus_pow_beta > max_delta_value) break;
+
+
+
+ cur_k1 = getCount(x_l_minusL,mask);
+ int lower_outlier_cost = cur_k1 * getBitWith(x_l_minus);
+
+ while ( end_value_i < unique_value_count) {
+// if(beta==3 && end_value_i==22)
+// {
+// System.out.println(x_l_minus);
+// System.out.println(x_l_plus);
+// }
+
+ int x_u_minus = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ if(x_u_minus < x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta){
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += lower_outlier_cost;
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus - x_l_plus);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = x_l_minus;
+ final_x_l_plus = x_l_plus;
+ final_k_end_value = x_u_plus;
+ final_x_u_minus = x_u_minus;
+ }
+ break;
+ }
+// else if (x_u_minus >= x_u_plus_pow_beta && x_u_plus >= x_u_plus_pow_beta) {
+// break;
+// }
+
+ end_value_i++;
+ }
+ }
+
+ }
+
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+// System.out.println(encode_pos);
+
+ return encode_pos;
+ }
+
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+ }
+ else {
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+ return encode_pos;
+ }
+
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += cur_delta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += currentDelta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ @Test
+ public void BOSDecodeImproveTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/bos_b";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/test";
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSEncodeImproveTest() throws IOException {
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/bos_b_improve";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/test";
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 200;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSVaryBlockSize() throws IOException, InterruptedException {
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/block_size_bos_improve";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+
+ int repeatTime2 = 100;
+// int[] file_i_list = {0,9,10};
+// for (int file_i = 9; file_i < 10; file_i++) {
+// for(int file_i :file_i_list){
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Block Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+
+// for (int block_size_i = 6; block_size_i > 5; block_size_i--) {
+ for (int block_size_i = 13; block_size_i > 5; block_size_i--) {
+ int block_size = (int) Math.pow(2, block_size_i);
+ System.out.println(block_size);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ compressed_size = BOSEncoderImprove(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-B-Improve",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(block_size_i),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+// Thread.sleep(10000);
+ }
+ writer.close();
+
+ }
+ }
+
+ @Test
+ public void BOSOUOTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/test";
+
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 200;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-B",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSHTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSHTest.java
deleted file mode 100644
index 59ad144..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSHTest.java
+++ /dev/null
@@ -1,1422 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-
-import static java.lang.Math.*;
-
-public class TSDIFFBOSHTest {
-
- public static long combine2Int(int int1, int int2) {
- return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
- }
-
- public static int getTime(long long1) {
- return ((int) (long1 >> 32));
- }
-
- public static int getValue(long long1) {
- return ((int) (long1));
- }
-
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
-
- public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
-
- public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx)<< (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos ++;
- bufIdx++;
- if (bufIdx >= width) {
- return ;
- }
- }
- }
-// return encode_pos;
- }
-
- public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add ((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size()-start) / 8;
- for(int i=0;i<block_num;i++){
- pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
- encode_pos +=bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values( encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
-
- }
- return result_list;
- }
-
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int[] min_delta,
- int supple_length) {
- int block_size = ts_block.length-1;
- int[] ts_block_delta = new int[ts_block.length+supple_length-1];
-
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- for (int i = 0; i < block_size; i++) {
-
- int epsilon_v = ts_block[i+1] - ts_block[i];
-
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
-
- }
- min_delta[0] = (ts_block[0]);
- min_delta[1] = (value_delta_min);
- min_delta[2] = (value_delta_max-value_delta_min);
-
- for (int i = 0; i < block_size; i++) {
- int epsilon_v = ts_block[i+1] - value_delta_min - ts_block[i];
- ts_block_delta[i] = epsilon_v;
- }
- for(int i = block_size;i<block_size+supple_length;i++){
- ts_block_delta[i] = 0;
- }
- return ts_block_delta;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta) {
- int[] ts_block_delta = new int[remaining-1];
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- int base = i*block_size+1;
- int end = i*block_size+remaining;
-
- int tmp_j_1 = ts_block[base-1];
- min_delta[0] =tmp_j_1;
- int j = base;
- int tmp_j;
-
- while(j<end){
- tmp_j = ts_block[j];
- int epsilon_v = tmp_j - tmp_j_1;
- ts_block_delta[j-base] = epsilon_v;
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
- tmp_j_1 = tmp_j;
- j++;
- }
- j = 0;
- end = remaining -1;
- while(j<end){
- ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
- j++;
- }
-
- min_delta[1] = value_delta_min;
- min_delta[2] = (value_delta_max-value_delta_min);
-
-
- return ts_block_delta;
- }
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- return encode_pos;
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (int) (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
-
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if ( cur_value<= final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
- k1++;
-
-
- } else if (cur_value >= final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value-1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
-
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
-
- int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
-
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
-
- int2Bytes(k_byte,encode_pos,cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);//final_left_max
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- }
- encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
- return encode_pos;
-
- }
-
- private static void calculateBits(int k1, int k2, int block_size, int alpha, int gamma, int k1_start, int k2_end, int min_bits, int[] result){
- int cur_bits = 0;
- cur_bits += Math.min((k1 + k2) * getBitWith(block_size - 1), block_size + k1 + k2);
- if(k1!=0)
- cur_bits += k1*alpha;
- if (k1 + k2 != block_size)
- cur_bits += (block_size - k1 - k2) *
- getBitWith(k2_end -k1_start - 2);
- if(k2!=0)
- cur_bits += k2 * gamma;
- if (cur_bits < min_bits) {
- result[0] = cur_bits;
- result[1] = k1_start;
- result[2] = k2_end;
- }
- }
-
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining, int encode_pos, byte[] cur_byte) {
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
-
- block_size = remaining - 1;
- int max_delta_value = min_delta[2];
-
- int boundary_index = (int) ((double) block_size / (double) (getBitWith(block_size-1)-1));
-
- int max_bit_width = getBitWith(max_delta_value) + 1;
- int[] lambda_list = {getBitWith(block_size-1),1};
- int lambda_0 = lambda_list[0];
-
- int count_x_min = 0;
- int count_x_max = 0;
- GroupL[] groupL = new GroupL[max_bit_width];
- GroupU[] groupU = new GroupU[max_bit_width];
- for (int i = 0; i < max_bit_width; i++) {
- int[] numbers = new int[block_size];
- int count = 0;
- groupL[i] = new GroupL(numbers, count, i);
- int[] numbersU = new int[block_size];
- groupU[i] = new GroupU(numbersU, count, i);
- }
- for (int value : ts_block_delta) {
- int alpha_i = getBitWith(value);
- if (value == 0) {
- count_x_min++;
- } else {
- groupL[alpha_i].addNumber(value); // (x_min+2^{alpha_i-1},x_min+2^{alpha_i})
- }
- int gamma_i = getBitWith(max_delta_value - value);
- if (value == max_delta_value) {
- count_x_max++;
- } else {
- groupU[gamma_i].addNumber(value);// (x_max-2^{gamma_i},x_max-2^{gamma_i-1})
- }
- }
- int[] alpha_list = new int[max_bit_width];
- int num_of_alpha_list = 0;
- int[] gamma_list = new int[max_bit_width];
- int num_of_gamma_list = 0;
- for (int i = 1; i < max_bit_width; i++) {
- if(groupL[i].count!=0){
- groupL[i].setCount_array();
- alpha_list[num_of_alpha_list] = i;
- num_of_alpha_list ++;
- }
- if(groupU[i].count!=0){
- groupU[i].setCount_array(max_delta_value);
- gamma_list[num_of_gamma_list] = i;
- num_of_gamma_list ++;
- }
- }
-
-
- int final_k_start_value = -1;//getUniqueValue(sorted_value_list[0], left_shift);
- int final_k_end_value = max_delta_value + 1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
-
- int alpha_size = getBitWith(max(max_delta_value-1,0));
-
- if( num_of_alpha_list ==0 && num_of_gamma_list==0){
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
- return encode_pos;
- }
-
-
- // -------------x_min-------------------
- // lower: x_min
- // upper: x_max
- int k1 = count_x_min;
- int k1_start = 0;
- int k2 = count_x_max;
- int k2_end = max_delta_value;
- int cur_bits = Math.min((k1 + k2) * getBitWith(block_size - 1), block_size + k1 + k2);
- cur_bits += k1;
- cur_bits += k2;
- cur_bits += (block_size - k1 - k2) * getBitWith(k2_end - k1_start - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
- // lower: x_min
- // upper: x_max - 1, x_max - 2, x_max - 4 ...
- //(x_max - 2^{gamma}, x_max - 2^{gamma-1} ]
-
- for (int gamma_i = 0; gamma_i < num_of_gamma_list-1; gamma_i++) {
-
- int gamma = gamma_list[gamma_i];
-
- int gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma-1} gamma : 1 to alpha_size-1
-
- GroupU cur_group_gamma = groupU[gamma];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
- int cur_gamma_count = cur_group_gamma.count;
-
- for(int unique_i=0;unique_i<gamma_unique_number;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;//min_upper_outlier
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- k2 += cur_gamma_count;
-
- }
-
- // lower: x_min
- // k2_end = x_max - 2^{alpha_size-1}
- // upper: (0,x_max - 2^{alpha_size-1}]
-
- int gap_gamma = (int) pow(2,alpha_size-1);
- k2_end = max_delta_value - gap_gamma;
- GroupU cur_group_gamma = groupU[alpha_size];
-
- long[] gamma_sorted = cur_group_gamma.sorted_value_list;
- int gamma_unique_number = cur_group_gamma.unique_number;
-
- for(int unique_i=0;unique_i<gamma_unique_number-1;unique_i++){
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
-
-
- cur_bits = 0;
- cur_bits += k1;
- cur_bits += Math.min((k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + k1 + unique_i_cur_k2);
-
- if (k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i - 2);
- cur_bits += unique_i_cur_k2 * alpha_size;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- // -------------x_min-------------------
-
-
-
-
- // [x_min+2^{alpha-1},x_min+2^{alpha}) alpha = 1 to alphasize - 1
- // -----------------[x_min+2^{alpha-1},x_min+2^{alpha})-------------------
- // alpha_size = getBitWith(max_delta_value);
-
- for (int alpha_i = 0; alpha_i < num_of_alpha_list-1; alpha_i++) {
-
- int alpha = alpha_list[alpha_i];
- int gap_alpha = (int) pow(2,alpha-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha];
-
-
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: x_max
-
- int alpha_value_count = cur_group_alpha.count;
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- // k1; count[x_min, x_min+2^{alpha-1}]
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int unique_i_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
- k2 = count_x_max;
- k2_end = max_delta_value;
- cur_bits = Math.min((unique_i_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_i_cur_k1 + k2);
- cur_bits += unique_i_cur_k1 * alpha;
- cur_bits += k2;
- if (unique_i_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_i_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
-
-
- int pow_2_alpha = (int) pow(2, alpha);
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
-
-
- int cur_k1 = k1; //count[x_min, x_min+2^{alpha-1}]
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]: k2_end - gap_gamma, k2_end
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- int gamma_size = (int) (log(max_delta_value - (int) pow(2, alpha)) / log(2));
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- int cur_k2 = k2; // count(x_max)
- int gamma_i = 0;
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- if(gamma > gamma_size){
- break;
- }
-
- // k1: count[x_min,x_min+2^{alpha-1}]
-
- gap_gamma = (int) pow(2, gamma - 1);
- k2_end = max_delta_value - gap_gamma;
- cur_group_gamma = groupU[gamma]; // (x_max - 2^{gamma}, x_max - 2^{gamma-1})
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
-
- int pow_2_gamma = gap_gamma*2;
-
- // prop 5.7
-
- int cur_k1_start = k1_start + gap_alpha - 1;
- int cur_k2_end = k2_end - gap_gamma + 1;
- int cur_k2_start_of_lambda = cur_k2 + gamma_value_count;// count[x_max - 2^{gamma}+1, x_max]
- int cur_k1_start_of_lambda = k1 + alpha_value_count; // count[x_min, 2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_start_of_lambda + cur_k2_start_of_lambda) * lambda_0, block_size + cur_k1_start_of_lambda + cur_k2_start_of_lambda); // (lambda * (cur_cur_k1 + cur_k2_start_of_lambda) + zeta);
- cur_bits += (cur_k1_start_of_lambda * alpha);
- cur_bits += (cur_k2_start_of_lambda * gamma);
- cur_bits += ((block_size - cur_k1_start_of_lambda - cur_k2_start_of_lambda) * getBitWith(cur_k2_end - cur_k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = cur_k1_start;
- final_k_end_value = cur_k2_end;
- }
-
- // prop 5.4
- int prop_5_4_k1 = cur_k1 + alpha_value_count; //count[x_min, x_min+2^{alpha}-1]
-
- cur_bits = 0;
- cur_bits += Math.min((prop_5_4_k1 + cur_k2) * lambda_0, block_size + prop_5_4_k1 + cur_k2); //(lambda * (prop_5_4_k1 + prop_5_4_k2) + zeta);
- cur_bits += (prop_5_4_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - prop_5_4_k1 - cur_k2) * getBitWith(k2_end - k1_start - gap_alpha - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + gap_alpha - 1;
- final_k_end_value = k2_end;
- }
-
- // prop 5.5
- int prop_5_5_k2 = cur_k2 + gamma_value_count; // count[x_max - 2^{gamma}+1,x_max}
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + prop_5_5_k2) * lambda_0, block_size + cur_k1 + prop_5_5_k2);// (lambda * (prop_5_5_k1 + prop_5_5_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (prop_5_5_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - prop_5_5_k2) * getBitWith(k2_end - gap_gamma - k1_start - 1));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end - gap_gamma + 1;
- }
-
- // prop 5.6
-
- cur_bits = 0;
- cur_bits += Math.min((cur_k1 + cur_k2) * lambda_0, block_size + cur_k1 + cur_k2);//(lambda * (prop_5_6_k1 + prop_5_6_k2) + zeta);
- cur_bits += (cur_k1 * alpha);
- cur_bits += (cur_k2 * gamma);
- cur_bits += ((block_size - cur_k1 - cur_k2) * getBitWith(k2_end - k1_start - 2));
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start;
- final_k_end_value = k2_end;
- }
-
-
-
- if ((max_pow_2_lambda + 1) * pow_2_alpha + pow_2_gamma <= max_delta_value
- && pow_2_alpha + (max_pow_2_lambda + 1) * pow_2_gamma <= max_delta_value) {
-
-
- }
- else {
-
- // lower: (x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}): k2_end - gap_gamma, k2_end
- // beta: getBitWidth(x_max - 2^{gamma} -(x_min+2^{alpha})) to getBitWidth(x_max - 2^{gamma-1}-(x_min+2^{alpha-1}) )
- int min_beta = getBitWith(max_delta_value-pow_2_gamma-pow_2_alpha);
- int max_beta = getBitWith(k2_end-k1_start);
-
-
-
- for(int beta = min_beta;beta <= max_beta;beta++) {
- int pow_2_beta = (int) pow(2, beta);
- if (beta >= alpha + max_lambda && beta <= gamma + 1 && pow_2_beta + pow_2_alpha + gap_gamma >= max_delta_value) {
-
- }
- else if (beta <= alpha + 1 && beta >= gamma + max_lambda && pow_2_beta + pow_2_alpha / 2 + pow_2_gamma >= max_delta_value) {
-
- }
- else if (beta <= alpha + 1 && beta <= gamma + 1 && pow_2_beta + pow_2_alpha / 2 + gap_gamma > max_delta_value) {
-
- } else {
- int cur_cur_k1;
- for (int unique_j = 0; unique_j < alpha_unique_number; unique_j++) {
- cur_cur_k1 = cur_k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);//alpha_unique_count_array[unique_j];// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);//alpha_unique_number_array[unique_j];
-
- int cur_cur_k2 ; // count[x_max - 2^{gamma-1}, x_max]
-
- int x_u_i_start = Math.max(0, k2_end - ( pow_2_beta + k1_start + x_l_i + 2));
- int x_u_i_end = Math.min(k2_end -( pow_2_beta/2 + k1_start + x_l_i + 2),gap_gamma-1);
-
-
- for (int unique_i = 0; unique_i < gamma_unique_number; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);//<=x_u_i_end
- if(x_u_i < x_u_i_start){
- continue;
- }
- if(x_u_i > x_u_i_end){
- break;
- }
- cur_cur_k2 = cur_k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
-
- cur_bits = 0;
- cur_bits += Math.min((cur_cur_k1 + cur_cur_k2) * lambda_0, block_size + cur_cur_k1 + cur_cur_k2);
- // (lambda * (cur_k1_start_of_lambda + cur_cur_k2) + zeta); // Math.min((cur_k1 + cur_cur_k2) * getBitWith(block_size - 1), block_size + cur_k1 + cur_k2);
- cur_bits += cur_cur_k1 * alpha;
- if (cur_cur_k1 + cur_cur_k2 != block_size)
- cur_bits += (block_size - cur_cur_k1 - cur_cur_k2) *
- getBitWith(k2_end - x_u_i - k1_start - x_l_i - 2);
- cur_bits += cur_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
-
- }
- }
- }
-
- }
-
- }
-
- cur_k2 = cur_k2 + gamma_value_count; // += count(x_max - 2^{gamma}, x_max - 2^{gamma-1})
-
- if(cur_k1+cur_k2>boundary_index){
- max_pow_2_lambda = (int) pow(2, 1);
- max_lambda = 1;
- }
-
- }
-
-
- k2 = cur_k2;
-
- // gamma: gamma_size+1 to ....
- // gamma size : x_min+2^{alpha} <= x_max - 2^{gamma} => 2^{gamma} <= x_max - 2^{alpha}
- //k2_end = x_max - 2^{gamma-1} ; gamma : 1 to gamma_size
- // lower: [x_min+2^{alpha-1},x_min+2^{alpha}) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}]
-
-
- for (; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- cur_group_gamma = groupU[gamma];
- k2_end = max_delta_value - gap_gamma;
-
- if(k2_end <= k1_start){
- break;
- }
- int gamma_value_count = cur_group_gamma.count;
-
- gamma_unique_number = cur_group_gamma.unique_number;
- gamma_sorted = cur_group_gamma.sorted_value_list;
-
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int unique_j_cur_k1 = cur_k1+ cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
-
-
- for (int unique_i = 0; unique_i < gamma_unique_number; unique_i++) {
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- if(k2_end - x_u_i <= k1_start+x_l_i){
- break;
- }
- int unique_i_cur_k2 = cur_k2+ cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
-
- }
- cur_k2 += gamma_value_count;
- }
-
- k1 += alpha_value_count;
- }
- // -------------------[x_min+2^{alpha-1},x_min+2^{alpha})-----------------
-
-
- // [x_min+2^{alpha_size-1},x_max)
- // ----------------[x_min+2^{alpha_size-1},x_max)--------------------
-
- // x_min+2^{alpha_size-1} < x_max && x_min+2^{alpha_size} >= x_max
- // alpha_size - 1 < log(xmax) && alpha_size >= log(xmax)
- // alpha_size = \lceil log(x_max-1+1) \rceil
-
- alpha_size = alpha_list[num_of_alpha_list-1];
- int gap_alpha = (int) pow(2,alpha_size-1);
- k1_start = gap_alpha;
- GroupL cur_group_alpha = groupL[alpha_size];
- int alpha_value_count = cur_group_alpha.count;
-
-
- // lower: [x_min+2^{alpha_size-1},x_max)
- // ----------------- [x_min+2^{alpha_size-1},x_max)----------------------
- long[] alpha_sorted = cur_group_alpha.sorted_value_list;
- int alpha_unique_number = cur_group_alpha.unique_number;
-
- int x_l_i_end = max_delta_value - k1_start; // [1, x_max - x_min+2^{alpha_size-1})
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
-
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- if (x_l_i >= x_l_i_end) {
- break;
- }
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
-
- // lower: [x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: x_max
- k2 = count_x_max;
- k2_end = max_delta_value;
-
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + k2);
- cur_bits += unique_j_cur_k1 * alpha_size;
- cur_bits += k2;
- if (unique_j_cur_k1 + k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - k2) * getBitWith(k2_end - k1_start - x_l_i - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end;
- }
- }
- // lower: (x_min+2^{alpha_size-1},x_max)
- // upper: (x_max - 2^{gamma}, x_max)
- // gamma: gamma_size+1 to ....
- // lower: (x_min+2^{alpha_size-1},x_max) : k1_start, k1_start + gap_alpha
- // upper: (x_max - 2^{gamma}, x_max - 2^{gamma-1}] : k2_end - gap_gamma, k2_end
-
- int max_pow_2_lambda = (int) pow(2, lambda_0);
- int max_lambda = lambda_0;
- for (int gamma_i = 0; gamma_i < num_of_gamma_list; gamma_i++) {
- int gamma = gamma_list[gamma_i];
- gap_gamma = (int) pow(2, gamma-1);
- k2_end = max_delta_value - gap_gamma; // x_max - 2^{gamma} ;
-
- if(k2_end<=k1_start){
- break;
- }
- cur_group_gamma = groupU[gamma];
-
- int gamma_value_count = cur_group_gamma.count;
- gamma_sorted = cur_group_gamma.sorted_value_list;
- gamma_unique_number = cur_group_gamma.unique_number;
-
- int pow_2_gamma = gap_gamma*2;
- for (int unique_j=0;unique_j<alpha_unique_number;unique_j++) {
- int x_l_i = cur_group_alpha.getUniqueValue(alpha_sorted[unique_j]);
- int x_u_i_end = k2_end - k1_start - x_l_i; // x_u_i>= k2_end - k1_start - x_l_i; break ;
-
- if(x_u_i_end <= 0 || x_l_i >= x_l_i_end){
- break;
- }
- int unique_j_cur_k1 = k1 + cur_group_alpha.getCount(alpha_sorted[unique_j]);// gamma_box_count_list[gamma];
- x_u_i_end = Math.min(x_u_i_end,gap_gamma);
-
-
- for(int unique_i=0;unique_i<gamma_unique_number;unique_i++){
-
- int x_u_i = cur_group_gamma.getUniqueValue(gamma_sorted[unique_i]);
- if(x_u_i>=x_u_i_end){
- break;
- }
-
- int unique_i_cur_k2 = k2 + cur_group_gamma.getCount(gamma_sorted[unique_i]);
- cur_bits = 0;
- cur_bits += Math.min((unique_j_cur_k1 + unique_i_cur_k2) * getBitWith(block_size - 1), block_size + unique_j_cur_k1 + unique_i_cur_k2);
- cur_bits += unique_j_cur_k1*alpha_size;
- if (unique_j_cur_k1 + unique_i_cur_k2 != block_size)
- cur_bits += (block_size - unique_j_cur_k1 - unique_i_cur_k2) *
- getBitWith(k2_end - x_u_i-k1_start-x_l_i - 2);
- cur_bits += unique_i_cur_k2 * gamma;
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k1_start + x_l_i;
- final_k_end_value = k2_end - x_u_i;
- }
- }
- }
- k2 += gamma_value_count;
-
-
-
- }
- // ------------------ [x_min+2^{alpha_size-1},x_max)--------------------
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
-
- return encode_pos;
- }
-
- public static int minNumberIndex(int alpha, int beta, int gamma){
- if(alpha<beta && beta < gamma){
- return 1; // prop 4.2
- } else if (alpha>beta && beta > gamma) {
- return 2;// prop 4.3
- } else if (beta<alpha && beta <gamma) {
- return 3;// prop 4.4
- }else{
- return 0; // alpha=beta=gamma
- }
- }
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
- block_size++;
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all,encode_pos,encoded_result);
- encode_pos += 4;
-
- int block_num = length_all / block_size;
- int2Bytes(block_size,encode_pos,encoded_result);
- encode_pos+= 4;
-
- for (int i = 0; i < block_num; i++) {
- encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- }
- else {
-
- int start = block_num * block_size;
- int remaining = length_all-start;
-
-
- encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
-
- }
-
-
- return encode_pos;
- }
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
-
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
- int value0 = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] =value0;
- value_pos_arr[0] ++;
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal= new ArrayList<>();;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
-
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size ) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
-
- for (int i = 0; i < block_size; i++) {
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = min_delta + final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- }
-
- pre_v = current_delta + pre_v;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
-
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
-
- int[] value_list = new int[length_all+block_size];
- block_size--;
-
- int[] value_pos_arr = new int[1];
- for (int k = 0; k < block_num; k++) {
-
-
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
-
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- remain_length --;
- BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
- }
- }
-
- @Test
- public void PruneBOSTest() throws IOException {
- String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/"; // your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/pruning_bos";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
- int repeatTime2 = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
-// f= tempList[1];
-
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
-
- String[] record = {
- f.toString(),
- "TS_2DIFF+BOS-P",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
- }
- writer.close();
-
- }
- }
-
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSLTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSLTest.java
deleted file mode 100644
index 7217efc..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSLTest.java
+++ /dev/null
@@ -1,1284 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-
-
-import static java.lang.Math.pow;
-
-public class TSDIFFBOSLTest {
-
- public static long combine2Int(int int1, int int2) {
- return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
- }
-
- public static int getTime(long long1) {
- return ((int) (long1 >> 32));
- }
-
- public static int getValue(long long1) {
- return ((int) (long1));
- }
-
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
-
- public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
-
- public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx)<< (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos ++;
- bufIdx++;
- if (bufIdx >= width) {
- return ;
- }
- }
- }
- }
-
- public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add ((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size()-start) / 8;
- for(int i=0;i<block_num;i++){
- pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
- encode_pos +=bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values( encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
-
- }
- return result_list;
- }
-
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int[] min_delta,
- int supple_length) {
- int block_size = ts_block.length-1;
- int[] ts_block_delta = new int[ts_block.length+supple_length-1];
-
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- for (int i = 0; i < block_size; i++) {
-
- int epsilon_v = ts_block[i+1] - ts_block[i];
-
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
-
- }
- min_delta[0] = (ts_block[0]);
- min_delta[1] = (value_delta_min);
- min_delta[2] = (value_delta_max-value_delta_min);
-
- for (int i = 0; i < block_size; i++) {
- int epsilon_v = ts_block[i+1] - value_delta_min - ts_block[i];
- ts_block_delta[i] = epsilon_v;
- }
- for(int i = block_size;i<block_size+supple_length;i++){
- ts_block_delta[i] = 0;
- }
- return ts_block_delta;
- }
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta) {
- int[] ts_block_delta = new int[remaining-1];
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- int base = i*block_size+1;
- int end = i*block_size+remaining;
-
- int tmp_j_1 = ts_block[base-1];
- min_delta[0] =tmp_j_1;
- int j = base;
- int tmp_j;
-
- while(j<end){
- tmp_j = ts_block[j];
- int epsilon_v = tmp_j - tmp_j_1;
- ts_block_delta[j-base] = epsilon_v;
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
- tmp_j_1 = tmp_j;
- j++;
- }
- j = 0;
- end = remaining -1;
- while(j<end){
- ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
- j++;
- }
-
- min_delta[1] = value_delta_min;
- min_delta[2] = (value_delta_max-value_delta_min);
-
-
- return ts_block_delta;
- }
-
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- return encode_pos;
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (int) (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
-
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if ( cur_value<= final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
- k1++;
-
-
- } else if (cur_value >= final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value-1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
-
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
-
- int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
-
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
-
- int2Bytes(k_byte,encode_pos,cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);//final_left_max
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- }
- encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
- return encode_pos;
-
- }
-
-
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
-
-
- block_size = remaining-1;
- int max_delta_value = min_delta[2];
- int max_bit = getBitWith(max_delta_value);
- int[] alpha_count_list = new int[max_bit+1];//count(xmin) count(xmin + 1) count(xmin + 2)... count(xmin + 8) ... count(xmin + 2^(i - 1))
- int[] alpha_box_count_list = new int[max_bit+1];// count(xmin, xmin + 1), count(xmin +1, xmin + 2)... count (xmin + 2^(i - 1), xmin + 2^i)
- int[] gamma_count_list = new int[max_bit+1];//count(xmax) count(xmax - 1)... count(xmax - 2^(i - 1))
- int[] gamma_box_count_list = new int[max_bit+1]; // count (xmax - 2^i, xmin - 2^(i-1))
- for(int value:ts_block_delta){
- if (value == 0){
- alpha_count_list[0]++;
- }
- else if (value == pow(2,getBitWith(value)-1)){
- alpha_count_list[getBitWith(value)]++;
- }else {
- alpha_box_count_list[getBitWith(value)]++;
- }
- if (value == max_delta_value){
- gamma_count_list[0]++;
- }
- else if (max_delta_value - value == pow(2,getBitWith(max_delta_value-value)-1)){
- gamma_count_list[getBitWith(max_delta_value-value)]++;
- }else {
- gamma_box_count_list[getBitWith(max_delta_value-value)]++;
- }
- }
-
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value -2) * (block_size));
- // lower: x_min
- // -------------------------------
- int cur_k1_close = alpha_count_list[0]; // lower: x_min
- int k_start_value_close = 0;
-
- int cur_bits;
- int alpha_2_pow = 1;
- int gamma_size = (int) (Math.log(max_delta_value - alpha_2_pow - 1)/Math.log(2));
- int cur_k2_close = gamma_count_list[0];
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int k_end_value_close = max_delta_value - (int) pow(2,gamma);
-
- cur_k2_close += gamma_count_list[gamma+1];
- cur_k2_close += gamma_box_count_list[gamma];
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- int k_end_value_open = k_end_value_close + 1;
-
- //C4 k1 close k2 open
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
-
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
- }
-
- // lower: x_min +1
- // -------------------------------
- cur_k1_close += alpha_count_list[1]; // lower outliers: x_min +1
- cur_k1_close += alpha_box_count_list[0];
- k_start_value_close = 1;
- cur_k2_close = gamma_count_list[0];
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int k_end_value_close = max_delta_value - (int) pow(2,gamma);
-
- cur_k2_close += gamma_count_list[gamma+1];
- cur_k2_close += gamma_box_count_list[gamma];
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- int k_end_value_open = k_end_value_close + 1;
-
- //C4 k1 close k2 open
- cur_bits = cur_k1_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
-
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
- }
- // lower: x_min + 2 ^alpha
- // -------------------------------
-
- for (int alpha = 1; alpha <= max_bit-1; alpha++) {
- //C1 k1 close k2 close
- k_start_value_close = (int) pow(2,alpha);
- int pow_2_alpha_i = alpha_count_list[alpha+1];
- int pow_2_alpha_i_1_to_2_alpha_i = alpha_box_count_list[alpha];
- cur_k1_close += pow_2_alpha_i;
- cur_k1_close += pow_2_alpha_i_1_to_2_alpha_i;
-
-
-
-
- alpha_2_pow = (int)pow(2,alpha);
- gamma_size = (int) (Math.log(max_delta_value - alpha_2_pow - 1)/Math.log(2));
-
- // upper outliers: x_max
- cur_k2_close = gamma_count_list[0];
- int k_end_value_close= max_delta_value;
-
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- int cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- int k_start_value_open = k_start_value_close - 1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha);
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- }
- // upper outliers: x_max -1
- cur_k2_close += gamma_count_list[1];
- cur_k2_close += gamma_box_count_list[0];
- k_end_value_close = max_delta_value-1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- }
-
- cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- k_start_value_open = k_start_value_close - 1;
- cur_bits=cur_k2_close;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * (alpha);
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- }
- // upper outliers: x_max -2^gamma
- for (int gamma = 1; gamma<=gamma_size;gamma++){
- int flag = 0;
- k_end_value_close = max_delta_value - (int) pow(2,gamma);
- cur_bits = 0;
- int pow_2_gamma_i = gamma_count_list[gamma+1];
- int pow_2_gamma_i_1_to_2_gamma_i = gamma_box_count_list[gamma];
- cur_k2_close += pow_2_gamma_i;
- cur_k2_close += pow_2_gamma_i_1_to_2_gamma_i;
- cur_bits += Math.min((cur_k1_close + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_close);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_close - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 1;
- }
-
- //C2 k1open k2open
- cur_bits = 0;
- cur_k1_open = cur_k1_close - alpha_count_list[alpha+1];
- int cur_k2_open = cur_k2_close - gamma_count_list[gamma+1];
- k_start_value_open = k_start_value_close - 1;
- int k_end_value_open = k_end_value_close + 1;
- cur_bits += Math.min((cur_k1_open + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_open);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * alpha;
- if (cur_k1_open + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_open - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 2;
- }
-
- //C3 k1 open k2 close
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_open + cur_k2_close) * getBitWith(block_size-1), block_size + cur_k1_open + cur_k2_close);
- if (cur_k1_open != 0)
- cur_bits += cur_k1_open * alpha;
- if (cur_k1_open + cur_k2_close != block_size)
- cur_bits += (block_size - cur_k1_open - cur_k2_close) * getBitWith(k_end_value_close - k_start_value_open - 2);
- if (cur_k2_close != 0)
- cur_bits += cur_k2_close * (gamma + 1);
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 3;
- }
-
- //C4 k1 close k2 open
- cur_bits = 0;
- cur_bits += Math.min((cur_k1_close + cur_k2_open) * getBitWith(block_size-1), block_size + cur_k1_close + cur_k2_open);
- if (cur_k1_close != 0)
- cur_bits += cur_k1_close * (alpha + 1);
- if (cur_k1_close + cur_k2_open != block_size)
- cur_bits += (block_size - cur_k1_close - cur_k2_open) * getBitWith(k_end_value_open - k_start_value_close - 2);
- if (cur_k2_open != 0)
- cur_bits += cur_k2_open * gamma;
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- flag = 4;
- }
- if(flag == 1){
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_close;
- } else if (flag == 2) {
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_open;
- } else if (flag == 3) {
- final_k_start_value = k_start_value_open;
- final_k_end_value = k_end_value_close;
- } else if (flag == 4) {
- final_k_start_value = k_start_value_close;
- final_k_end_value = k_end_value_open;
- }
-
- }
- }
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
-
- return encode_pos;
- }
-
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
- block_size++;
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all,encode_pos,encoded_result);
- encode_pos += 4;
-
- int block_num = length_all / block_size;
- int2Bytes(block_size,encode_pos,encoded_result);
- encode_pos+= 4;
-
- for (int i = 0; i < block_num; i++) {
- encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- }
- else {
-
- int start = block_num * block_size;
- int remaining = length_all-start;
-
-
- encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
-
- }
-
-
- return encode_pos;
- }
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
-
-
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
- int value0 = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] =value0;
- value_pos_arr[0] ++;
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal= new ArrayList<>();;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
-
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size ) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
-
- for (int i = 0; i < block_size; i++) {
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = min_delta + final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- }
-
- pre_v = current_delta + pre_v;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
-
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
-
- int[] value_list = new int[length_all+block_size];
- block_size--;
-
- int[] value_pos_arr = new int[1];
- for (int k = 0; k < block_num; k++) {
-
-
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
-
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- remain_length --;
- BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
- }
- }
-
- @Test
- public void BWSTest() throws IOException {
- String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/"; // your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/bws";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
- int repeatTime2 = 100;
-
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
-
- String[] record = {
- f.toString(),
- "TS_2DIFF+BWS",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-
- }
- writer.close();
-
- }
- }
-
- @Test
- public void BOSVaryBlockSize() throws IOException {
- String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/"; // your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/block_size_bws";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-
- int repeatTime2 = 500;
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Block Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
-
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
-
- for (int block_size_i = 13; block_size_i > 4; block_size_i--) {
- int block_size = (int) Math.pow(2, block_size_i);
- System.out.println(block_size);
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- compressed_size = BOSEncoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
- String[] record = {
- f.toString(),
- "TS_2DIFF+BOS-L",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(block_size_i),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-
- }
-
- }
- writer.close();
-
- }
- }
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSMImproveTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSMImproveTest.java
new file mode 100644
index 0000000..e218148
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSMImproveTest.java
@@ -0,0 +1,3016 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
+import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile;
+import java.nio.channels.Channels;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.utils.SeekableInMemoryByteChannel;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static java.lang.Math.*;
+import static org.apache.iotdb.tsfile.constant.TestConstant.random;
+
+public class TSDIFFBOSMImproveTest {
+
+ public static long combine2Int(int int1, int int2) {
+ return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
+ }
+
+ public static int getTime(long long1) {
+ return ((int) (long1 >> 32));
+ }
+
+ public static int getValue(long long1) {
+ return ((int) (long1));
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+// return encode_pos;
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+
+ }
+ return result_list;
+ }
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int[] min_delta,
+ int supple_length) {
+ int block_size = ts_block.length-1;
+ int[] ts_block_delta = new int[ts_block.length+supple_length-1];
+
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ for (int i = 0; i < block_size; i++) {
+
+ int epsilon_v = ts_block[i+1] - ts_block[i];
+
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+
+ }
+ min_delta[0] = (ts_block[0]);
+ min_delta[1] = (value_delta_min);
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+ for (int i = 0; i < block_size; i++) {
+ int epsilon_v = ts_block[i+1] - value_delta_min - ts_block[i];
+ ts_block_delta[i] = epsilon_v;
+ }
+ for(int i = block_size;i<block_size+supple_length;i++){
+ ts_block_delta[i] = 0;
+ }
+ return ts_block_delta;
+ }
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining-1];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i*block_size+1;
+ int end = i*block_size+remaining;
+
+ int tmp_j_1 = ts_block[base-1];
+ min_delta[0] =tmp_j_1;
+ int j = base;
+ int tmp_j;
+
+ while(j<end){
+ tmp_j = ts_block[j];
+ int epsilon_v = tmp_j - tmp_j_1;
+ ts_block_delta[j-base] = epsilon_v;
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ tmp_j_1 = tmp_j;
+ j++;
+ }
+ j = 0;
+ end = remaining -1;
+ while(j<end){
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+ j++;
+ }
+
+ min_delta[1] = value_delta_min;
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+
+ return ts_block_delta;
+ }
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (int) (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+ final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+ final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+ final_normal.add(cur_value - final_x_l_plus);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+
+ if(k1==0 && k2==0){
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// return encode_pos;
+ }
+ else{
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+ bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+ }
+
+
+// if(k1+k2!=block_size)
+ encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ if (k1 != 0)
+ encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+ if (k2 != 0)
+ encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+ private static void calculateBits(int k1, int k2, int block_size, int alpha, int gamma, int k1_start, int k2_end, int min_bits, int[] result){
+ int cur_bits = 0;
+ cur_bits += Math.min((k1 + k2) * getBitWith(block_size - 1), block_size + k1 + k2);
+ if(k1!=0)
+ cur_bits += k1*alpha;
+ if (k1 + k2 != block_size)
+ cur_bits += (block_size - k1 - k2) *
+ getBitWith(k2_end -k1_start - 2);
+ if(k2!=0)
+ cur_bits += k2 * gamma;
+ if (cur_bits < min_bits) {
+ result[0] = cur_bits;
+ result[1] = k1_start;
+ result[2] = k2_end;
+ }
+ }
+
+
+ public static int findMedian(int[] arr) {
+ if (arr == null || arr.length == 0) {
+ throw new IllegalArgumentException("数组不能为空");
+ }
+ int n = arr.length;
+ if (n > 400){
+ return quickSelect(arr, 0, 99, 50);
+ }else if (n > 100){
+ return quickSelect(arr, 0, (n - 1)/4, n / 8);
+ }else {
+ return quickSelect(arr, 0, n - 1, n / 2);
+ }
+ }
+
+// private static int quickSelect(int[] arr, int left, int right, int k){
+// int pivotV=arr[left+random.nextInt(right-left+1)],tmpV;
+// int posEqual=left,posSmaller=left; // a[left,posEqual): = pivotV; a[posEqual,posSmaller): < pivotV
+// for(int i=left;i<=right;i++){
+// if(arr[i]==pivotV){
+// tmpV=arr[i];
+// arr[i]=arr[posSmaller];
+// arr[posSmaller]=arr[posEqual];
+// arr[posEqual]=tmpV;
+// posEqual++;
+// posSmaller++;
+// }else
+// if(arr[i]<pivotV){
+// tmpV=arr[posSmaller];
+// arr[posSmaller]=arr[i];
+// arr[i]=tmpV;
+// posSmaller++;
+// }
+// }
+// if(k+(posEqual-left)<=posSmaller-1)return quickSelect(arr,posEqual,posSmaller-1,k+(posEqual-left));
+// else if(k<=posSmaller-1)return pivotV;
+// else return quickSelect(arr,posSmaller,right,k);
+// }
+ private static int quickSelect(int[] arr, int left, int right, int k) {
+ if (left == right) {
+ return arr[left];
+ }
+ if (areAllElementsEqual(arr, left, right)) {
+ return arr[left];
+ }
+ int pivotIndex = left + random.nextInt(right - left + 1);
+ swap(arr, pivotIndex, right);
+
+ pivotIndex = partition(arr, left, right);
+ if (abs(pivotIndex - k) < (right - left)/6) {
+ return arr[k];
+ } else if (k <= pivotIndex - (right - left)/6) {
+ return quickSelect(arr, left, pivotIndex - 1, k);
+ } else {
+ return quickSelect(arr, pivotIndex + 1, right, k);
+ }
+ }
+ private static boolean areAllElementsEqual(int[] arr, int left, int right) {
+ for (int i = left + 1; i <= right; i++) {
+ if (arr[i] != arr[left]) {
+ return false;
+ }
+ }
+ return true;
+ }
+ private static int partition(int[] arr, int left, int right) {
+ int pivot = arr[right];
+ int i = left;
+ for (int j = left; j < right; j++) {
+ if (arr[j] <= pivot) {
+ swap(arr, i, j);
+ i++;
+ }
+ }
+ swap(arr, i, right);
+ return i;
+ }
+
+ private static void swap(int[] arr, int i, int j) {
+ int temp = arr[i];
+ arr[i] = arr[j];
+ arr[j] = temp;
+ }
+
+ public static long compress(byte[] data) throws IOException {
+ File file = new File("example.7z");
+ file.createNewFile();
+ Path path = Paths.get("example.7z");
+ SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ, StandardOpenOption.WRITE);
+ SevenZOutputFile szof = new SevenZOutputFile(channel);
+ SevenZArchiveEntry entry = szof.createArchiveEntry(null, "entryName");
+ szof.putArchiveEntry(entry);
+ szof.write(data);
+ szof.closeArchiveEntry();
+ return channel.size();
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBitsImprove(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining, int encode_pos, byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+// System.out.println(Arrays.toString(ts_block_delta));
+
+ block_size = remaining - 1;
+ int max_delta_value = min_delta[2];
+
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedian(findMedianArray);
+
+
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value <=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+// min_bits +=;
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+ }
+
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = 1 << (beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+// if(min_bits == bp_cost){
+//
+// }
+// else
+// System.out.println(min_bits/8);
+ encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+
+ return encode_pos;
+ }
+
+ public static int minNumberIndex(int alpha, int beta, int gamma){
+ if(alpha<beta && beta < gamma){
+ return 1; // prop 4.2
+ } else if (alpha>beta && beta > gamma) {
+ return 2;// prop 4.3
+ } else if (beta<alpha && beta <gamma) {
+ return 3;// prop 4.4
+ }else{
+ return 0; // alpha=beta=gamma
+ }
+ }
+
+ private static void addToArchiveCompression(SevenZOutputFile out, File file, String dir) {
+ String name = dir + File.separator + file.getName();
+ if(dir.equals(".")) {
+ name = file.getName();
+ }
+ if (file.isFile()){
+ SevenZArchiveEntry entry = null;
+ FileInputStream in = null;
+ try {
+ entry = out.createArchiveEntry(file, name);
+ out.putArchiveEntry(entry);
+ in = new FileInputStream(file);
+ byte[] b = new byte[1024];
+ int count = 0;
+ while ((count = in.read(b)) > 0) {
+ out.write(b, 0, count);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ out.closeArchiveEntry();
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ } else if (file.isDirectory()) {
+ File[] children = file.listFiles();
+ if (children != null){
+ for (File child : children){
+ addToArchiveCompression(out, child, name);
+ }
+ }
+ } else {
+ System.out.println(file.getName() + " is not supported");
+ }
+ }
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+
+// for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < block_num; i++) {
+// int start_encode_pos = encode_pos;
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos-start_encode_pos);
+// System.out.println("------------------------------------------");
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ }
+ else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+ private static int BOSBlockEncoderImprove(int[] ts_block, int block_i, int block_size, int remaining, int encode_pos, byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+// System.out.println(Arrays.toString(ts_block_delta));
+
+ block_size = remaining - 1;
+ int max_delta_value = min_delta[2];
+
+
+ int max_bit_width = getBitWith(max_delta_value) + 1;
+
+
+ int[] findMedianArray = new int[block_size];
+ System.arraycopy(ts_block_delta, 0, findMedianArray, 0, block_size);
+
+ int median = findMedian(findMedianArray);
+
+
+
+ // int xl= median;
+ // int xu = median;
+ // xl = 2 median - xu
+ // xl = xu - 2 ^ beta
+ int left_number = 0;
+ int right_number = 0;
+
+ int length_outlier = block_size;
+// for(int value:findMedianArray){
+// if(value <=median) left_number++;
+// if (value >= median) right_number ++;
+// }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = (getBitWith(final_k_end_value - final_k_start_value - 2) * (block_size));
+// min_bits +=;
+
+
+ int[] count_left = new int[max_bit_width];
+ int[] count_right = new int[max_bit_width];
+ int count_0 = 0;
+
+
+ for(int i=0;i<length_outlier;i++){
+ int cur_value = ts_block_delta[i];
+ if(cur_value > median){
+ int beta = getBitWith(cur_value - median) ;
+ count_right[beta] ++;
+ } else if (cur_value < median) {
+ int beta = getBitWith(median - cur_value) ;
+ count_left[beta] ++;
+ }else{
+ count_0 ++;
+ }
+ }
+
+
+ for(int beta = max_bit_width - 1; beta > 0 ; beta --){
+ left_number += count_left[beta];
+ right_number += count_right[beta];
+ int pow_beta = 1<< (beta-1);
+ int xu = min(max_delta_value+1, median + pow_beta) ;
+ int xl = max(median - pow_beta,-1);
+ int cur_bits = Math.min((left_number + right_number) * getBitWith(block_size - 1), block_size + left_number + right_number);
+ cur_bits += left_number * getBitWith(xl);
+ cur_bits += right_number * getBitWith(max_delta_value - xu);
+ cur_bits += (block_size - left_number - right_number) * getBitWith(xu - xl - 2);
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = xl;
+ final_x_l_plus = xl + 1;
+ final_k_end_value = xu;
+ final_x_u_minus = xu -1;
+ }
+
+ }
+// if(min_bits == bp_cost){
+//
+// }
+// else
+// System.out.println(min_bits/8);
+ encode_pos = BOSEncodeBitsImprove(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+
+
+ return encode_pos;
+ }
+
+ public static int BOSEncoderImprove(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+// int[] test_block = new int[block_size];
+// System.arraycopy(data, 0, test_block, 0, block_size);
+// System.out.println(Arrays.toString(test_block));
+// for (int i = 0; i < 1; i++) {
+ for (int i = 0; i < block_num; i++) {
+// int start_encode_pos = encode_pos;
+ encode_pos = BOSBlockEncoderImprove(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos-start_encode_pos);
+// System.out.println("------------------------------------------");
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ }
+ else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+
+
+ encode_pos = BOSBlockEncoderImprove(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+ int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ if(k1!=0 || k2 != 0){
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+ }else {
+ bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ }
+
+
+
+
+ ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+ final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+
+ decode_pos = decode_pos_normal.get(0);
+ if (k1 != 0) {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+ decode_pos = decode_pos_result_left.get(0);
+ }
+ if (k2 != 0) {
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+ decode_pos = decode_pos_result_right.get(0);
+ }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+
+ for (int i = 0; i < block_size; i++) {
+ int current_delta;
+ if (left_outlier_i >= k1) {
+ if (right_outlier_i >= k2) {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ }
+ } else if (i == final_left_outlier_index.get(left_outlier_i)) {
+ current_delta = min_delta + final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ } else {
+
+ if (right_outlier_i >= k2) {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ } else if (i == final_right_outlier_index.get(right_outlier_i)) {
+ current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
+ right_outlier_i++;
+ } else {
+ current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
+ normal_i++;
+ }
+ }
+
+ pre_v = current_delta + pre_v;
+ value_list[value_pos_arr[0]] = pre_v;
+ value_pos_arr[0]++;
+ }
+ return decode_pos;
+ }
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoderImprove(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += cur_delta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += currentDelta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+ public static void BOSDecoderImprove(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+
+ decode_pos = BOSBlockDecoderImprove(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoderImprove(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+// int[] test_block = new int[block_size];
+// System.arraycopy(value_list, 0, test_block, 0, block_size);
+// System.out.println(Arrays.toString(test_block));
+// System.out.println(value_list[1000]);
+ }
+
+ @Test
+ public void BOSImproveEncodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/bos_m_improve";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/test";
+// String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+
+// for (int file_i = 0; file_i < 1; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f= tempList[1];
+
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+ writer.close();
+
+ }
+ }
+ @Test
+ public void BOSImproveDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSPreviousDecodeTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 9; file_i < 10; file_i++) {
+//
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void compressBOSTest() throws IOException {
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/bos_m_comp";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 5;
+// for (int file_i = 8; file_i < 9; file_i++) {
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+// for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+// "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ double ratio = 0;
+ double compressed_size = 0;
+ long compressTime = encodeTime;
+ s = System.nanoTime();
+ byte[] compressed = new byte[0];
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ BZip2CompressorOutputStream bzip2Out = new BZip2CompressorOutputStream(baos);
+ bzip2Out.write(encoded_result);
+ bzip2Out.finish();
+ compressed = baos.toByteArray();
+ }
+ e = System.nanoTime();
+ compressTime += ((e - s) / repeatTime2);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime2; repeat++)
+// BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "BOS+"+ "7-Zip",
+ String.valueOf(compressTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void compressBOSTest2() throws IOException {
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/bos_m_lz4_comp";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 500;
+// for (int file_i = 8; file_i < 9; file_i++) {
+ CompressionType[] compressList = {
+ CompressionType.LZ4,
+ };
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+// for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+// System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+// "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+
+ for (CompressionType comp : compressList) {
+ double ratio = 0;
+ double compressed_size = 0;
+ long compressTime = encodeTime;
+// compressTime += ((e - s) / repeatTime2);
+ System.out.println(compressTime);
+
+ s = System.nanoTime();
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ byte[] compressed = new byte[0];
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ compressed = compressor.compress(encoded_result);
+ }
+ e = System.nanoTime();
+ compressTime += ((e - s) / repeatTime2);
+
+ System.out.println(compressTime-encodeTime);
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+// for (int repeat = 0; repeat < repeatTime2; repeat++)
+// BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "BOS+"+ comp,
+ String.valueOf(compressTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+// System.out.println(ratio);
+ }
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void ExpTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/exp_test";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("Synthetic_Exp_100");
+ dataset_name.add("Synthetic_Exp_1000");
+ dataset_name.add("Synthetic_Exp_10000");
+ dataset_name.add("Synthetic_Exp_100000");
+ dataset_name.add("Synthetic_Exp_1000000");
+ dataset_name.add("Synthetic_Normal_100");
+ dataset_name.add("Synthetic_Normal_1000");
+ dataset_name.add("Synthetic_Normal_10000");
+ dataset_name.add("Synthetic_Normal_100000");
+ dataset_name.add("Synthetic_Normal_1000000");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/Exp_100.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Exp_1000.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Exp_10000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_100000.csv");// 2
+ output_path_list.add(output_parent_dir + "/Exp_1000000.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_100.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Normal_1000.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Normal_10000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_100000.csv");//5
+ output_path_list.add(output_parent_dir + "/Normal_1000000.csv");//5
+// dataset_block_size.add(2048);
+
+ int repeatTime2 = 100;
+
+// for (int file_i = 0; file_i < 1; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ if(f.toString().contains(".DS")) continue;
+// f= tempList[1];
+
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+ writer.close();
+
+ }
+ }
+
+ @Test
+ public void BOSVaryBlockSize() throws IOException {
+ String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+// String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/block_size_bos_m";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/block_size_bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+
+ int repeatTime2 = 100;
+// int[] file_i_list = {0,9,10};
+// for (int file_i = 9; file_i < 10; file_i++) {
+// for(int file_i :file_i_list){
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Block Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+
+// for (int block_size_i = 5; block_size_i < 14; block_size_i++) {
+ for (int block_size_i = 13; block_size_i > 4; block_size_i--) {
+// for (int block_size_i = 10; block_size_i > 9; block_size_i--) {
+ int block_size = (int) Math.pow(2, block_size_i);
+ System.out.println(block_size);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ compressed_size = BOSEncoderImprove(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoderImprove(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(block_size_i),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+
+ }
+ writer.close();
+
+ }
+ }
+
+
+ @Test
+ public void BOSQueryTest() throws IOException, InterruptedException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O2_query_processing/time/bos_m";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 500;
+// for (int file_i = 1; file_i < 2; file_i++) {
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Insert Time",
+ "Query Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+// Thread.sleep(5000);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ String file_bin_str = "icde0802/supply_experiment/R2O2_query_processing/time/str_1.bin";
+ long input_time = 0;
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ long start_encode = System.nanoTime();
+ length = BOSEncoderImprove(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ long end_encode = System.nanoTime();
+ try (FileOutputStream fos = new FileOutputStream(parent_dir +file_bin_str)) {
+ // 只写入前length个元素
+ fos.write(encoded_result, 0, length);
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ long end_io = System.nanoTime();
+ encodeTime += (end_encode - start_encode);
+ input_time += (end_io - end_encode);
+ }
+
+
+ long e = System.nanoTime();
+ encodeTime = (encodeTime / repeatTime2);
+ input_time = (input_time / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+ long output_time = 0;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++){
+ long start_io = System.nanoTime();
+
+ try (FileInputStream fis = new FileInputStream(parent_dir +file_bin_str)) {
+ // 读取数据到byte数组中
+ int bytesRead = fis.read(encoded_result);
+// System.out.println("Number of bytes read: " + bytesRead);
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ long start_decode = System.nanoTime();
+ BOSDecoderImprove(encoded_result);
+ long end_decode = System.nanoTime();
+ decodeTime += (end_decode - start_decode);
+ output_time += (start_decode - start_io);
+ }
+
+ e = System.nanoTime();
+ decodeTime = (decodeTime / repeatTime2);
+ output_time = (output_time / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(input_time),
+ String.valueOf(output_time),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSMOUOTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O5_skew_distribution/compression_ratio/bos_m_improve";
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/test";
+// String output_parent_dir = parent_dir + "icde0802/supply_experiment/R2O3_lower_outlier_compare/compression_ratio/bos";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 200;
+
+// for (int file_i = 6; file_i < 7; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+ if(file_i == 4 ) repeatTime2 = 1;
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f= tempList[1];
+
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-M",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+ writer.close();
+
+ }
+ }
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVImproveDecodeTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVImproveDecodeTest.java
new file mode 100644
index 0000000..3aa69b3
--- /dev/null
+++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVImproveDecodeTest.java
@@ -0,0 +1,1895 @@
+package org.apache.iotdb.tsfile.encoding;
+
+import com.csvreader.CsvReader;
+import com.csvreader.CsvWriter;
+import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
+import org.apache.commons.compress.archivers.sevenz.SevenZOutputFile;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.compress.IUnCompressor;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.encoding.encoder.Encoder;
+import org.apache.iotdb.tsfile.encoding.encoder.FloatEncoder;
+import org.apache.iotdb.tsfile.encoding.encoder.TSEncodingBuilder;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static java.lang.Math.pow;
+//import static org.apache.iotdb.tsfile.encoding.CompressTest.addToArchiveCompression;
+
+public class TSDIFFBOSVImproveDecodeTest {
+
+ public static long combine2Int(int int1, int int2) {
+ return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
+ }
+
+ public static int getTime(long long1) {
+ return ((int) (long1 >> 32));
+ }
+
+ public static int getValue(long long1) {
+ return ((int) (long1));
+ }
+
+ public static int getCount(long long1, int mask) {
+ return ((int) (long1 & mask));
+ }
+ public static int getUniqueValue(long long1, int left_shift) {
+ return ((int) ((long1) >> left_shift));
+ }
+
+ public static int getBitWith(int num) {
+ if (num == 0) return 1;
+ else return 32 - Integer.numberOfLeadingZeros(num);
+ }
+
+ public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+
+ public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer);
+ }
+
+ private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
+ cur_byte[encode_pos] = (byte) (integer >> 24);
+ cur_byte[encode_pos+1] = (byte) (integer >> 16);
+ cur_byte[encode_pos+2] = (byte) (integer >> 8);
+ cur_byte[encode_pos+3] = (byte) (integer);
+ }
+
+ public static int bytes2Integer(byte[] encoded, int start, int num) {
+ int value = 0;
+ if (num > 4) {
+ System.out.println("bytes2Integer error");
+ return 0;
+ }
+ for (int i = 0; i < num; i++) {
+ value <<= 8;
+ int b = encoded[i + start] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
+ long value = 0;
+ for (int i = 0; i < 4; i++) {
+ value <<= 8;
+ int b = encoded[i + decode_pos] & 0xFF;
+ value |= b;
+ }
+ return value;
+ }
+
+ public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
+
+ while (valueIdx < 8 + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
+
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values.get(valueIdx) << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
+
+ while (leftSize >= width && valueIdx < 8 + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values.get(valueIdx)<< (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < 8 + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the
+ // buffer
+ buffer |= (values.get(valueIdx) >>> (width - leftSize));
+ leftBit = width - leftSize;
+ }
+
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ encode_pos ++;
+ bufIdx++;
+ if (bufIdx >= width) {
+ return ;
+ }
+ }
+ }
+// return encode_pos;
+ }
+
+ public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e.,
+ // number of available bits to be decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
+
+ while (valueIdx < 8) {
+ // If current available bits are not enough to decode one Integer,
+ // then add next byte from buf to 'buffer' until totalBits >= width
+ while (totalBits < width) {
+ buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
+ byteIdx++;
+ totalBits += 8;
+ }
+
+ // If current available bits are enough to decode one Integer,
+ // then decode one Integer one by one until left bits in 'buffer' is
+ // not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ result_list.add ((int) (buffer >>> (totalBits - width)));
+ valueIdx++;
+ totalBits -= width;
+ buffer = buffer & ((1L << totalBits) - 1);
+ }
+ }
+ }
+
+ public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
+ int block_num = (numbers.size()-start) / 8;
+ for(int i=0;i<block_num;i++){
+ pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
+ encode_pos +=bit_width;
+ }
+
+ return encode_pos;
+
+ }
+
+ public static ArrayList<Integer> decodeBitPacking(
+ byte[] encoded, int decode_pos, int bit_width, int block_size) {
+ ArrayList<Integer> result_list = new ArrayList<>();
+ int block_num = (block_size - 1) / 8;
+
+ for (int i = 0; i < block_num; i++) { // bitpacking
+ unpack8Values( encoded, decode_pos, bit_width, result_list);
+ decode_pos += bit_width;
+
+ }
+ return result_list;
+ }
+
+
+ public static int[] getAbsDeltaTsBlock(
+ int[] ts_block,
+ int i,
+ int block_size,
+ int remaining,
+ int[] min_delta) {
+ int[] ts_block_delta = new int[remaining-1];
+
+ int value_delta_min = Integer.MAX_VALUE;
+ int value_delta_max = Integer.MIN_VALUE;
+ int base = i*block_size+1;
+ int end = i*block_size+remaining;
+
+ int tmp_j_1 = ts_block[base-1];
+ min_delta[0] =tmp_j_1;
+ int j = base;
+ int tmp_j;
+
+ while(j<end){
+ tmp_j = ts_block[j];
+ int epsilon_v = tmp_j - tmp_j_1;
+ ts_block_delta[j-base] = epsilon_v;
+ if (epsilon_v < value_delta_min) {
+ value_delta_min = epsilon_v;
+ }
+ if (epsilon_v > value_delta_max) {
+ value_delta_max = epsilon_v;
+ }
+ tmp_j_1 = tmp_j;
+ j++;
+ }
+ j = 0;
+ end = remaining -1;
+ while(j<end){
+ ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
+ j++;
+ }
+
+ min_delta[1] = value_delta_min;
+ min_delta[2] = (value_delta_max-value_delta_min);
+
+
+ return ts_block_delta;
+ }
+
+
+ public static int encodeOutlier2Bytes(
+ ArrayList<Integer> ts_block_delta,
+ int bit_width,
+ int encode_pos, byte[] encoded_result) {
+
+ encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
+
+ int n_k = ts_block_delta.size();
+ int n_k_b = n_k / 8;
+ long cur_remaining = 0; // encoded int
+ int cur_number_bits = 0; // the bit width used of encoded int
+ for (int i = n_k_b * 8; i < n_k; i++) {
+ long cur_value = ts_block_delta.get(i);
+ int cur_bit_width = bit_width; // remaining bit width of current value
+
+ if (cur_number_bits + bit_width >= 32) {
+ cur_remaining <<= (32 - cur_number_bits);
+ cur_bit_width = bit_width - 32 + cur_number_bits;
+ cur_remaining += ((cur_value >> cur_bit_width));
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ cur_remaining = 0;
+ cur_number_bits = 0;
+ }
+
+ cur_remaining <<= cur_bit_width;
+ cur_number_bits += cur_bit_width;
+ cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
+ }
+ cur_remaining <<= (32 - cur_number_bits);
+ long2intBytes(cur_remaining,encode_pos,encoded_result);
+ encode_pos += 4;
+ return encode_pos;
+
+ }
+
+
+ public static ArrayList<Integer> decodeOutlier2Bytes(
+ byte[] encoded,
+ int decode_pos,
+ int bit_width,
+ int length,
+ ArrayList<Integer> encoded_pos_result
+ ) {
+
+ int n_k_b = length / 8;
+ int remaining = length - n_k_b * 8;
+ ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
+ decode_pos += n_k_b * bit_width;
+
+ ArrayList<Long> int_remaining = new ArrayList<>();
+ int int_remaining_size = remaining * bit_width / 32 + 1;
+ for (int j = 0; j < int_remaining_size; j++) {
+ int_remaining.add(bytesLong2Integer(encoded, decode_pos));
+ decode_pos += 4;
+ }
+
+ int cur_remaining_bits = 32; // remaining bit width of current value
+ long cur_number = int_remaining.get(0);
+ int cur_number_i = 1;
+ for (int i = n_k_b * 8; i < length; i++) {
+ if (bit_width < cur_remaining_bits) {
+ int tmp = (int) (cur_number >> (32 - bit_width));
+ result_list.add(tmp);
+ cur_number <<= bit_width;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits -= bit_width;
+ } else {
+ int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
+ int remain_bits = bit_width - cur_remaining_bits;
+ tmp <<= remain_bits;
+
+ cur_number = int_remaining.get(cur_number_i);
+ cur_number_i++;
+ tmp += (int) (cur_number >> (32 - remain_bits));
+ result_list.add(tmp);
+ cur_number <<= remain_bits;
+ cur_number &= 0xFFFFFFFFL;
+ cur_remaining_bits = 32 - remain_bits;
+ }
+ }
+ encoded_pos_result.add(decode_pos);
+ return result_list;
+ }
+
+ public static int EncodeBits(int num,
+ int bit_width,
+ int encode_pos,
+ byte[] cur_byte,
+ int[] bit_index_list){
+ // 找到要插入的位的索引
+ int bit_index = bit_index_list[0] ;//cur_byte[encode_pos + 1];
+
+ // 计算数值的起始位位置
+ int remaining_bits = bit_width;
+
+ while (remaining_bits > 0) {
+ // 计算在当前字节中可以使用的位数
+ int available_bits = bit_index;
+ int bits_to_write = Math.min(available_bits, remaining_bits);
+
+ // 更新 bit_index
+ bit_index = available_bits - bits_to_write;
+
+ // 计算要写入的位的掩码和数值
+ int mask = (1 << bits_to_write) - 1;
+ int bits = (num >> (remaining_bits - bits_to_write)) & mask;
+
+ // 写入到当前位置
+ cur_byte[encode_pos] &= (byte) ~(mask << bit_index); // 清除对应位置的位
+ cur_byte[encode_pos] |= (byte) (bits << bit_index);
+
+ // 更新位宽和数值
+ remaining_bits -= bits_to_write;
+ if (bit_index == 0) {
+ bit_index = 8;
+ encode_pos++;
+ }
+ }
+ bit_index_list[0] = bit_index;
+// cur_byte[encode_pos + 1] = (byte) bit_index;
+ return encode_pos;
+ }
+ private static int BOSEncodeBits(int[] ts_block_delta,
+ int final_k_start_value,
+ int final_x_l_plus,
+ int final_k_end_value,
+ int final_x_u_minus,
+ int max_delta_value,
+ int[] min_delta,
+ int encode_pos,
+ byte[] cur_byte) {
+ int block_size = ts_block_delta.length;
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_left_outlier = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier = new ArrayList<>();
+ ArrayList<Integer> final_normal = new ArrayList<>();
+ int k1 = 0;
+ int k2 = 0;
+
+
+
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int index_bitmap_outlier = 0;
+ int cur_index_bitmap_outlier_bits = 0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_value = ts_block_delta[i];
+ if ( cur_value<= final_k_start_value) {
+// encode_pos = EncodeBits(cur_value,left_bit_width,encode_pos,cur_byte);
+// final_left_outlier.add(cur_value);
+ final_left_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 3;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 1;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k1++;
+
+
+ } else if (cur_value >= final_k_end_value) {
+// encode_pos = EncodeBits(cur_value- final_k_end_value,right_bit_width,encode_pos,cur_byte);
+// final_right_outlier.add(cur_value - final_k_end_value);
+ final_right_outlier_index.add(i);
+ if (cur_index_bitmap_outlier_bits % 8 != 7) {
+ index_bitmap_outlier <<= 2;
+ index_bitmap_outlier += 2;
+ cur_index_bitmap_outlier_bits += 2;
+ } else {
+ index_bitmap_outlier <<= 1;
+ index_bitmap_outlier += 1;
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ cur_index_bitmap_outlier_bits = 1;
+ }
+ k2++;
+
+ } else {
+// final_normal.add(cur_value - final_x_l_plus);
+// encode_pos = EncodeBits(cur_value- final_x_l_plus,right_bit_width,encode_pos,cur_byte);
+ index_bitmap_outlier <<= 1;
+ cur_index_bitmap_outlier_bits += 1;
+ }
+ if (cur_index_bitmap_outlier_bits % 8 == 0) {
+ bitmap_outlier.add(index_bitmap_outlier);
+ index_bitmap_outlier = 0;
+ }
+ }
+ if (cur_index_bitmap_outlier_bits % 8 != 0) {
+
+ index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
+
+ index_bitmap_outlier &= 0xFF;
+ bitmap_outlier.add(index_bitmap_outlier);
+ }
+
+ int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
+
+
+ int k_byte = (k1 << 1);
+ k_byte += final_alpha;
+ k_byte += (k2 << 16);
+
+ int2Bytes(k_byte,encode_pos,cur_byte);
+ encode_pos += 4;
+
+
+ int2Bytes(min_delta[0],encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(min_delta[1],encode_pos,cur_byte);
+ encode_pos += 4;
+
+ int bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+ intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+ encode_pos += 1;
+ int[] bit_index_list = new int[1];
+ bit_index_list[0] = 8;
+
+ if(final_k_start_value<0 && final_k_end_value > max_delta_value){
+// int bit_width_final= getBitWith(final_x_u_minus - final_x_l_plus);
+// cur_byte[encode_pos+1] = 8;
+ for (int cur_value : ts_block_delta) {
+ encode_pos = EncodeBits(cur_value, bit_width_final, encode_pos, cur_byte, bit_index_list);
+// final_normal.add(cur_value);
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+// cur_byte[encode_pos+1] = 0;
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+ return encode_pos;
+ }
+
+
+ int left_bit_width = getBitWith(final_k_start_value);//final_left_max
+ int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
+ int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+ encode_pos += 4;
+ int2Bytes(final_k_end_value,encode_pos,cur_byte);
+ encode_pos += 4;
+
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+ intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+ intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+ encode_pos += 1;
+
+ if (final_alpha == 0) { // 0
+
+ for (int i : bitmap_outlier) {
+
+ intByte2Bytes(i,encode_pos,cur_byte);
+ encode_pos += 1;
+ }
+ } else {
+ encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+ }
+// cur_byte[encode_pos+1] = 8;
+// bit_index_list[0] = 8;
+ for (int cur_value : ts_block_delta) {
+ if (cur_value <= final_k_start_value) {
+ encode_pos = EncodeBits(cur_value, left_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else if (cur_value >= final_k_end_value) {
+ encode_pos = EncodeBits(cur_value - final_k_end_value, right_bit_width, encode_pos, cur_byte,bit_index_list);
+ } else {
+ encode_pos = EncodeBits(cur_value - final_x_l_plus, bit_width_final, encode_pos, cur_byte,bit_index_list);
+ }
+ }
+ if(bit_index_list[0] != 8){
+ encode_pos ++;
+ }
+
+// cur_byte[encode_pos+1] = 0;
+
+// if(k1==0 && k2==0){
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+//
+//
+// }
+// else{
+// int2Bytes(final_x_l_plus,encode_pos,cur_byte);
+// encode_pos += 4;
+// int2Bytes(final_k_end_value,encode_pos,cur_byte);
+// encode_pos += 4;
+//
+// bit_width_final = getBitWith(final_x_u_minus - final_x_l_plus);
+// intByte2Bytes(bit_width_final,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(left_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// intByte2Bytes(right_bit_width,encode_pos,cur_byte);
+// encode_pos += 1;
+// if (final_alpha == 0) { // 0
+//
+// for (int i : bitmap_outlier) {
+//
+// intByte2Bytes(i,encode_pos,cur_byte);
+// encode_pos += 1;
+// }
+// } else {
+// encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
+// }
+// }
+
+
+// if(k1+k2!=block_size)
+// encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
+// if (k1 != 0)
+// encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
+// if (k2 != 0)
+// encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
+ return encode_pos;
+
+ }
+
+
+ private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
+
+ int[] min_delta = new int[3];
+ int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
+
+
+ block_size = remaining-1;
+ int max_delta_value = min_delta[2];
+ int[] value_list = new int[block_size];
+ int unique_value_count = 0;
+ int[] value_count_list = new int[max_delta_value+1];
+ for(int value:ts_block_delta){
+ if(value_count_list[value]==0){
+ value_count_list[value] = 1;
+ value_list[unique_value_count] = value;
+ unique_value_count ++;
+ }else{
+ value_count_list[value] ++;
+ }
+ }
+
+ int left_shift = getBitWith(block_size);
+ int mask = (1 << left_shift) - 1;
+ long[] sorted_value_list = new long[unique_value_count];
+ int count = 0;
+
+ for(int i=0;i<unique_value_count;i++){
+ int value = value_list[i];
+ sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
+ }
+ Arrays.sort(sorted_value_list);
+
+ for(int i=0;i<unique_value_count;i++){
+ count += getCount(sorted_value_list[i], mask);
+ sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
+ }
+
+
+ int final_k_start_value = -1; // x_l_minus
+ int final_x_l_plus = 0; // x_l_plus
+ int final_k_end_value = max_delta_value+1; // x_u_plus
+ int final_x_u_minus = max_delta_value; // x_u_minus
+
+ int min_bits = 0;
+ min_bits += (getBitWith(final_k_end_value - final_k_start_value - 2 ) * (block_size));
+
+ int cur_k1 = 0;
+
+ int x_l_plus_value = 0; // x_l_plus
+ int x_u_minus_value = max_delta_value; // x_u_plus
+
+ for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ int x_u_plus_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+ int cur_bits = 0;
+ int cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value); // cur_k1 = 0
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - x_u_plus_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_x_u_minus = x_u_minus_value;
+ final_k_end_value = x_u_plus_value;
+ }
+ }
+
+ int k_start_value = -1; // x_l_minus
+
+ for (int start_value_i = 0; start_value_i < unique_value_count-1; start_value_i++) {
+ long k_start_valueL = sorted_value_list[start_value_i];
+ k_start_value = getUniqueValue(k_start_valueL, left_shift) ;
+ x_l_plus_value = getUniqueValue(sorted_value_list[start_value_i+1], left_shift) ;
+
+
+ cur_k1 = getCount(k_start_valueL,mask);
+
+ int k_end_value;
+ int cur_bits;
+ int cur_k2;
+ k_end_value = max_delta_value + 1;
+
+ cur_bits = 0;
+ cur_k2 = 0;
+ cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1) * getBitWith(k_end_value- x_l_plus_value); //cur_k2 =0
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = max_delta_value;
+ }
+
+ for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
+
+ x_u_minus_value = getUniqueValue(sorted_value_list[end_value_i-1], left_shift);
+ k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
+
+ cur_bits = 0;
+ cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
+
+ cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
+ cur_bits += cur_k1 * getBitWith(k_start_value);
+ if (cur_k1 + cur_k2 != block_size)
+ cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(x_u_minus_value - x_l_plus_value);
+ if (cur_k2 != 0)
+ cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
+
+
+ if (cur_bits < min_bits) {
+ min_bits = cur_bits;
+
+ final_k_start_value = k_start_value;
+ final_x_l_plus = x_l_plus_value;
+ final_k_end_value = k_end_value;
+ final_x_u_minus = x_u_minus_value;
+ }
+
+ }
+ }
+// System.out.println(min_bits/4);
+ encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_x_l_plus, final_k_end_value, final_x_u_minus,
+ max_delta_value, min_delta, encode_pos , cur_byte);
+// System.out.println(encode_pos);
+// System.out.println(encode_pos);
+ return encode_pos;
+ }
+
+
+ public static int BOSEncoder(
+ int[] data, int block_size, byte[] encoded_result) {
+ block_size++;
+
+ int length_all = data.length;
+
+ int encode_pos = 0;
+ int2Bytes(length_all,encode_pos,encoded_result);
+ encode_pos += 4;
+
+ int block_num = length_all / block_size;
+ int2Bytes(block_size,encode_pos,encoded_result);
+ encode_pos+= 4;
+
+ for (int i = 0; i < block_num; i++) {
+// int start_encode_pos = encode_pos;
+ encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
+// System.out.println(encode_pos-start_encode_pos);
+// System.out.println("------------------------------------------");
+ }
+
+ int remaining_length = length_all - block_num * block_size;
+ if (remaining_length <= 3) {
+ for (int i = remaining_length; i > 0; i--) {
+ int2Bytes(data[data.length - i], encode_pos, encoded_result);
+ encode_pos += 4;
+ }
+
+ }
+ else {
+
+ int start = block_num * block_size;
+ int remaining = length_all-start;
+ encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
+
+ }
+
+
+ return encode_pos;
+ }
+
+
+ public static int DecodeBits(byte[] cur_byte, int bit_width, int[] decode_pos_list) {
+ int decode_pos = decode_pos_list[0];
+ int bit_index = decode_pos_list[1]; //cur_byte[decode_pos + 1];
+ int remaining_bits = bit_width;
+ int num = 0;
+
+ while (remaining_bits > 0) {
+ int available_bits = bit_index;
+ int bits_to_read = Math.min(available_bits, remaining_bits);
+
+ // 计算要读取的位的掩码
+ int mask = (1 << bits_to_read) - 1;
+ int bits = (cur_byte[decode_pos] >> (available_bits - bits_to_read)) & mask;
+
+ // 将读取的位合并到结果中
+ num = (num << bits_to_read) | bits;
+
+ // 更新位宽和 bit_index
+ remaining_bits -= bits_to_read;
+ bit_index = available_bits - bits_to_read;
+
+ if (bit_index == 0) {
+ bit_index = 8;
+ decode_pos++;
+ }
+ }
+ decode_pos_list[0] = decode_pos;
+ decode_pos_list[1] = bit_index;
+
+ return num;
+ }
+ public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
+
+ int k_byte = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int k1_byte = (int) (k_byte % pow(2, 16));
+ int k1 = k1_byte / 2;
+ int final_alpha = k1_byte % 2;
+
+ int k2 = (int) (k_byte / pow(2, 16));
+
+ int value0 = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] =value0;
+ value_pos_arr[0] ++;
+
+ int min_delta = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ int[] decode_list = new int[2];
+ decode_list[0]= decode_pos;
+ decode_list[1]= 8;
+
+ int valuePos = value_pos_arr[0];
+
+ if(k1==0 && k2==0){
+ int pre_v = value0;
+ for (int i = 0; i < block_size; i++) {
+ int cur_delta = min_delta + DecodeBits(encoded, bit_width_final, decode_list);
+ pre_v += cur_delta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ return decode_list[0];
+ }
+
+ ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
+ ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
+// ArrayList<Integer> final_left_outlier = new ArrayList<>();
+// ArrayList<Integer> final_right_outlier = new ArrayList<>();
+// ArrayList<Integer> final_normal= new ArrayList<>();;
+ ArrayList<Integer> bitmap_outlier = new ArrayList<>();
+ int final_k_start_value = 0;
+ int final_k_end_value = 0;
+// int bit_width_final = 0;
+ int left_bit_width = 0;
+ int right_bit_width = 0;
+
+ final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+ final_k_end_value = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+// bit_width_final = bytes2Integer(encoded, decode_pos, 1);
+// decode_pos += 1;
+
+ left_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+ right_bit_width = bytes2Integer(encoded, decode_pos, 1);
+ decode_pos += 1;
+
+ if (final_alpha == 0) {
+ int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
+ for (int i = 0; i < bitmap_bytes; i++) {
+ bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
+ decode_pos += 1;
+ }
+ int bitmap_outlier_i = 0;
+ int remaining_bits = 8;
+ int tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ int i = 0;
+ while (i < block_size ) {
+ if (remaining_bits > 1) {
+ int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ if (remaining_bits == 0) {
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ }
+ } else if (remaining_bits == 1) {
+ int bit_i = tmp & 0x1;
+ remaining_bits = 8;
+ if (bitmap_outlier_i >= bitmap_bytes) break;
+ tmp = bitmap_outlier.get(bitmap_outlier_i);
+ bitmap_outlier_i++;
+ if (bit_i == 1) {
+ int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
+ remaining_bits -= 1;
+ if (bit_left_right == 1) {
+ final_left_outlier_index.add(i);
+ } else {
+ final_right_outlier_index.add(i);
+ }
+ }
+ }
+ i++;
+ }
+ } else {
+ ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+ final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
+ decode_pos = (decode_pos_result_left.get(0));
+ ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+ final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
+ decode_pos = (decode_pos_result_right.get(0));
+ }
+
+
+
+
+
+// ArrayList<Integer> decode_pos_normal = new ArrayList<>();
+// final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
+//
+// decode_pos = decode_pos_normal.get(0);
+// if (k1 != 0) {
+// ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
+// final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
+// decode_pos = decode_pos_result_left.get(0);
+// }
+// if (k2 != 0) {
+// ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
+// final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
+// decode_pos = decode_pos_result_right.get(0);
+// }
+ int left_outlier_i = 0;
+ int right_outlier_i = 0;
+ int normal_i = 0;
+ int pre_v = value0;
+// int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
+
+// Precompute constants
+ int normalOffset = min_delta + final_k_start_value;
+ int rightOutlierOffset = min_delta + final_k_end_value;
+
+// Initialize indices and pre-fetch next outlier positions
+ int leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ int rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ decode_list[0]= decode_pos;
+// decode_list[1]= 8;
+ // Use a local variable for the position
+ for (int i = 0; i < block_size; i++) {
+ int currentDelta;
+ if (i == leftOutlierNextIndex) {
+ // Process left outlier
+ currentDelta = min_delta + DecodeBits(encoded, left_bit_width, decode_list); // final_left_outlier.get(left_outlier_i);
+ left_outlier_i++;
+ leftOutlierNextIndex = (left_outlier_i < k1) ? final_left_outlier_index.get(left_outlier_i) : Integer.MAX_VALUE;
+ } else if (i == rightOutlierNextIndex) {
+ // Process right outlier
+ currentDelta = rightOutlierOffset + DecodeBits(encoded, right_bit_width, decode_list);// final_right_outlier.get(right_outlier_i);
+ right_outlier_i++;
+ rightOutlierNextIndex = (right_outlier_i < k2) ? final_right_outlier_index.get(right_outlier_i) : Integer.MAX_VALUE;
+ } else {
+ // Process normal value
+ currentDelta = normalOffset + DecodeBits(encoded, bit_width_final, decode_list);
+ normal_i++;
+ }
+
+ // Update the cumulative value and store it
+ pre_v += currentDelta;
+ value_list[valuePos++] = pre_v;
+ }
+ value_pos_arr[0] = valuePos;
+ if(decode_list[1]!=8){
+ return decode_list[0]+1;
+ }else {
+ return decode_list[0];
+ }
+// decode_pos = decode_list[0];
+// Update the position in the array
+
+
+// return decode_pos;
+ }
+
+ public static void BOSDecoder(byte[] encoded) {
+
+ int decode_pos = 0;
+ int length_all = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ int block_size = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+
+
+
+ int block_num = length_all / block_size;
+ int remain_length = length_all - block_num * block_size;
+
+
+ int[] value_list = new int[length_all+block_size];
+ block_size--;
+
+ int[] value_pos_arr = new int[1];
+ for (int k = 0; k < block_num; k++) {
+
+
+ decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
+
+ }
+
+ if (remain_length <= 3) {
+ for (int i = 0; i < remain_length; i++) {
+ int value_end = bytes2Integer(encoded, decode_pos, 4);
+ decode_pos += 4;
+ value_list[value_pos_arr[0]] = value_end;
+ value_pos_arr[0]++;
+ }
+ } else {
+ remain_length --;
+ BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
+ }
+ }
+
+ @Test
+ public void BOSOptimalTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R1O4_decode_time/compression_ratio/bos_v";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 1; file_i < 2; file_i++) {
+
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+// f=tempList[2];
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+// String value = loader.getValues()[index];
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+// data.add(Integer.valueOf(value));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ compressed_size += length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-V",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+// break;
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void BOSVaryBlockSize() throws IOException, InterruptedException {
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/"; // your data path
+// String output_parent_dir = parent_dir + "icde0802/compression_ratio/block_size_bos_v";
+ String output_parent_dir = parent_dir + "icde0802/compression_ratio/block_size_bos_v";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+
+ int repeatTime2 = 20;
+ int[] file_i_list = {0,9,10};
+// for (int file_i = 9; file_i < 10; file_i++) {
+// for(int file_i :file_i_list){
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Block Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ Thread.sleep(10000);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+
+ loader.readHeaders();
+
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+
+// for (int block_size_i = 5; block_size_i <14; block_size_i++) {
+ for (int block_size_i = 13; block_size_i > 4; block_size_i--) {
+ int block_size = (int) Math.pow(2, block_size_i);
+ System.out.println(block_size);
+ long encodeTime = 0;
+ long decodeTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ compressed_size = BOSEncoder(data2_arr, block_size, encoded_result);
+ }
+
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+ String[] record = {
+ f.toString(),
+ "TS_2DIFF+BOS-V",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(block_size_i),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+
+ }
+
+ }
+ writer.close();
+
+ }
+ }
+
+ private static void addToArchiveCompression(SevenZOutputFile out, File file, String dir) {
+ String name = dir + File.separator + file.getName();
+ if(dir.equals(".")) {
+ name = file.getName();
+ }
+ if (file.isFile()){
+ SevenZArchiveEntry entry = null;
+ FileInputStream in = null;
+ try {
+ entry = out.createArchiveEntry(file, name);
+ out.putArchiveEntry(entry);
+ in = new FileInputStream(file);
+ byte[] b = new byte[1024];
+ int count = 0;
+ while ((count = in.read(b)) > 0) {
+ out.write(b, 0, count);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ out.closeArchiveEntry();
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ }
+ } else if (file.isDirectory()) {
+ File[] children = file.listFiles();
+ if (children != null){
+ for (File child : children){
+ addToArchiveCompression(out, child, name);
+ }
+ }
+ } else {
+ System.out.println(file.getName() + " is not supported");
+ }
+ }
+
+ @Test
+ public void compressBOSTest() throws IOException {
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";// your data path
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/bos_v_comp";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ ArrayList<Integer> dataset_block_size = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+ for (String value : dataset_name) {
+ input_path_list.add(input_parent_dir + value);
+ dataset_block_size.add(1024);
+ }
+
+ output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
+// dataset_block_size.add(256);
+ output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
+// dataset_block_size.add(1024);
+ output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
+// dataset_block_size.add(2048);
+ output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
+// dataset_block_size.add(1024);
+
+ int repeatTime2 = 100;
+// for (int file_i = 8; file_i < 9; file_i++) {
+ CompressionType[] compressList = {
+ CompressionType.LZ4,
+ CompressionType.LZMA2,
+ };
+
+ for (int file_i = input_path_list.size()-1; file_i >=0 ; file_i--) {
+
+ String inputPath = input_path_list.get(file_i);
+ System.out.println(inputPath);
+ String Output = output_path_list.get(file_i);
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+// "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<Integer> data1 = new ArrayList<>();
+ ArrayList<Integer> data2 = new ArrayList<>();
+
+ loader.readHeaders();
+ while (loader.readRecord()) {
+ data1.add(Integer.valueOf(loader.getValues()[0]));
+ data2.add(Integer.valueOf(loader.getValues()[1]));
+ }
+ inputStream.close();
+
+ int[] data2_arr = new int[data1.size()];
+ for(int i = 0;i<data2.size();i++){
+ data2_arr[i] = data2.get(i);
+ }
+ byte[] encoded_result = new byte[data2_arr.length*4];
+ long encodeTime = 0;
+ long decodeTime = 0;
+ int length = 0;
+
+ long s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++) {
+ length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
+ }
+ long e = System.nanoTime();
+ encodeTime += ((e - s) / repeatTime2);
+
+ for (CompressionType comp : compressList) {
+ double ratio = 0;
+ double compressed_size = 0;
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ byte[] compressed = compressor.compress(encoded_result);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+ s = System.nanoTime();
+ for (int repeat = 0; repeat < repeatTime2; repeat++)
+ BOSDecoder(encoded_result);
+ e = System.nanoTime();
+ decodeTime += ((e - s) / repeatTime2);
+
+
+ String[] record = {
+ f.toString(),
+ "BOS+"+ comp,
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ double ratio = 0;
+ double compressed_size = 0;
+ File outfile = new File(parent_dir + "icde0802/example.bin");
+
+ // 使用FileOutputStream将byte数组写入文件
+ try (FileOutputStream fos = new FileOutputStream(outfile)) {
+ fos.write(encoded_result);
+ } catch (IOException e2) {
+ // 处理可能的I/O异常
+ e2.printStackTrace();
+ }
+
+ File input = new File(parent_dir + "icde0802/example.bin");
+ File output = new File(parent_dir + "icde0802/example.7z");
+ SevenZOutputFile out = new SevenZOutputFile(output);
+
+ addToArchiveCompression(out, input, ".");
+ out.closeArchiveEntry();
+
+ long compressed = output.length();
+
+
+ // test compression ratio and compressed size
+ compressed_size += compressed;
+ double ratioTmp =
+ (double) compressed / (double) (double) (data1.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+
+ String[] record = {
+ f.toString(),
+ "BOS+7-Zip",
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data1.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+ };
+ writer.writeRecord(record);
+ System.out.println(ratio);
+ }
+ writer.close();
+ }
+ }
+
+ @Test
+ public void compressionBPTest() throws IOException {
+
+ String parent_dir = "/Users/xiaojinzhao/Documents/GitHub/encoding-outlier/";
+// String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
+ String output_parent_dir = parent_dir + "icde0802/supply_experiment/R3O2_compare_compression/compression_ratio/bp_comp/";
+ String input_parent_dir = parent_dir + "trans_data/";
+ ArrayList<String> input_path_list = new ArrayList<>();
+ ArrayList<String> output_path_list = new ArrayList<>();
+ ArrayList<String> dataset_name = new ArrayList<>();
+ dataset_name.add("CS-Sensors");
+ dataset_name.add("Metro-Traffic");
+ dataset_name.add("USGS-Earthquakes");
+ dataset_name.add("YZ-Electricity");
+ dataset_name.add("GW-Magnetic");
+ dataset_name.add("TY-Fuel");
+ dataset_name.add("Cyber-Vehicle");
+ dataset_name.add("Vehicle-Charge");
+ dataset_name.add("Nifty-Stocks");
+ dataset_name.add("TH-Climate");
+ dataset_name.add("TY-Transport");
+ dataset_name.add("EPM-Education");
+
+
+ for (String item : dataset_name) {
+ input_path_list.add(input_parent_dir + item);
+ }
+
+ output_path_list.add(output_parent_dir + "CS-Sensors_ratio.csv"); // 0
+
+ output_path_list.add(output_parent_dir + "Metro-Traffic_ratio.csv");// 1
+
+ output_path_list.add(output_parent_dir + "USGS-Earthquakes_ratio.csv");// 2
+
+ output_path_list.add(output_parent_dir + "YZ-Electricity_ratio.csv"); // 3
+
+ output_path_list.add(output_parent_dir + "GW-Magnetic_ratio.csv"); //4
+
+ output_path_list.add(output_parent_dir + "TY-Fuel_ratio.csv");//5
+
+ output_path_list.add(output_parent_dir + "Cyber-Vehicle_ratio.csv"); //6
+
+ output_path_list.add(output_parent_dir + "Vehicle-Charge_ratio.csv");//7
+
+ output_path_list.add(output_parent_dir + "Nifty-Stocks_ratio.csv");//8
+
+ output_path_list.add(output_parent_dir + "TH-Climate_ratio.csv");//9
+
+ output_path_list.add(output_parent_dir + "TY-Transport_ratio.csv");//10
+
+ output_path_list.add(output_parent_dir + "EPM-Education_ratio.csv");//11
+
+// for (int file_i = 7; file_i < 8; file_i++) {
+ for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
+ String inputPath = input_path_list.get(file_i);
+ String Output = output_path_list.get(file_i);
+
+ // speed
+ int repeatTime = 100; // set repeat time
+ String dataTypeName = "int"; // set dataType
+
+ File file = new File(inputPath);
+ File[] tempList = file.listFiles();
+
+ // select encoding algorithms
+ TSEncoding[] encodingList = {
+ TSEncoding.PLAIN ,
+// TSEncoding.TS_2DIFF,
+// TSEncoding.RLE,
+// TSEncoding.SPRINTZ,
+// TSEncoding.GORILLA,
+// TSEncoding.RLBE,
+// TSEncoding.CHIMP,
+// TSEncoding.BUFF
+ };
+
+ CompressionType[] compressList = {
+// CompressionType.UNCOMPRESSED,
+ CompressionType.LZ4,
+ CompressionType.LZMA2,
+
+ };
+ CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
+
+ String[] head = {
+ "Input Direction",
+ "Encoding Algorithm",
+// "Compress Algorithm",
+ "Encoding Time",
+ "Decoding Time",
+ "Points",
+ "Compressed Size",
+ "Compression Ratio"
+// "Input Direction",
+// "Column Index",
+// "Encoding Algorithm",
+// "Compress Algorithm",
+// "Encoding Time",
+// "Decoding Time",
+// "Compress Time",
+// "Uncompress Time",
+// "Points",
+// "Compressed Size",
+// "Compression Ratio"
+ };
+ writer.writeRecord(head); // write header to output file
+
+ assert tempList != null;
+ ArrayList<Integer> columnIndexes = new ArrayList<>(); // set the column indexes of compressed
+ for (int i = 0; i < 2; i++) {
+ columnIndexes.add(i, i);
+ }
+ for (File f : tempList) {
+ System.out.println(f);
+ InputStream inputStream = Files.newInputStream(f.toPath());
+ CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ ArrayList<String> data = new ArrayList<>();
+
+ for (int index : columnIndexes) {
+ if (index == 0){
+ continue;
+ }
+ int max_precision = 2;
+ if (file_i == 3){
+ max_precision = 4;
+ } else if (file_i == 4 ||file_i == 5) {
+ max_precision = 3;
+ }
+ loader.readHeaders();
+ data.clear();
+ while (loader.readRecord()) {
+ String v = loader.getValues()[index];
+
+ data.add(v);
+ if (!v.matches("-?\\d+")){
+ dataTypeName = "float";
+ }
+ }
+ inputStream.close();
+
+ switch (dataTypeName) {
+ case "int":
+ {
+ TSDataType dataType = TSDataType.INT32; // set TSDataType
+ ArrayList<Integer> tmp = new ArrayList<>();
+ for (String value : data) {
+ tmp.add(Integer.valueOf(value));
+ }
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder =
+ TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+ long encodeTime = 0;
+ long decodeTime = 0;
+
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+
+ double ratio = 0;
+ double compressed_size = 0;
+
+ long compressTime = 0;
+ long uncompressTime = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (int val : tmp) {
+ encoder.encode(val, buffer);
+ }
+
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Integer.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ s = System.nanoTime();
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readInt(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+
+ String[] record = {
+ f.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+// f.toString(),
+// String.valueOf(index),
+// encoding.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// String.valueOf(decodeTime),
+// String.valueOf(compressTime),
+// String.valueOf(uncompressTime),
+// String.valueOf(data.size()),
+// String.valueOf(compressed_size),
+// String.valueOf(ratio)
+ };
+ System.out.println(ratio);
+ writer.writeRecord(record);
+ }
+ }
+ tmp.clear();
+ break;
+ }
+
+ // write info to file
+ case "float":
+ {
+ System.out.println("get float");
+ TSDataType dataType = TSDataType.FLOAT;
+ ArrayList<Float> tmp = new ArrayList<>();
+ data.removeIf(String::isEmpty);
+ for (String datum : data) {
+ tmp.add(Float.valueOf(datum));
+ }
+
+ // Iterate over each encoding algorithm
+ for (TSEncoding encoding : encodingList) {
+ Encoder encoder;
+ encoder = TSEncodingBuilder.getEncodingBuilder(encoding).getEncoder(dataType);
+ if (encoding == TSEncoding.TS_2DIFF){
+ encoder = new FloatEncoder(TSEncoding.TS_2DIFF, dataType, max_precision);
+ } else if (encoding == TSEncoding.RLE){
+ encoder = new FloatEncoder(TSEncoding.RLE, dataType, max_precision);
+ }
+
+ Decoder decoder = Decoder.getDecoderByType(encoding, dataType);
+
+ long encodeTime = 0;
+ long decodeTime = 0;
+ // Iterate over each compression algorithm
+ for (CompressionType comp : compressList) {
+ ICompressor compressor = ICompressor.getCompressor(comp);
+ IUnCompressor unCompressor = IUnCompressor.getUnCompressor(comp);
+ long compressTime = 0;
+ long uncompressTime = 0;
+ double ratio = 0;
+ double compressed_size = 0;
+
+ // repeat many times to test time
+ for (int i = 0; i < repeatTime; i++) {
+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+
+ // test encode time
+ long s = System.nanoTime();
+ for (float val : tmp) {
+ encoder.encode(val, buffer);
+ }
+ encoder.flush(buffer);
+ long e = System.nanoTime();
+ encodeTime += (e - s);
+
+ // test compress time
+ byte[] elems = buffer.toByteArray();
+ s = System.nanoTime();
+ byte[] compressed = compressor.compress(elems);
+ e = System.nanoTime();
+ compressTime += (e - s);
+
+ // test compression ratio and compressed size
+ compressed_size += compressed.length;
+ double ratioTmp =
+ (double) compressed.length / (double) (tmp.size() * Float.BYTES);
+ ratio += ratioTmp;
+
+ // test uncompress time
+ s = System.nanoTime();
+ unCompressor.uncompress(compressed);
+ e = System.nanoTime();
+ uncompressTime += (e - s);
+
+ // test decode time
+ ByteBuffer ebuffer = ByteBuffer.wrap(buffer.toByteArray());
+ while (decoder.hasNext(ebuffer)) {
+ decoder.readFloat(ebuffer);
+ }
+ e = System.nanoTime();
+ decodeTime += (e - s);
+
+ buffer.close();
+ }
+ ratio /= repeatTime;
+ compressed_size /= repeatTime;
+ encodeTime /= repeatTime;
+ decodeTime /= repeatTime;
+
+ // write info to file
+ String[] record = {
+ f.toString(),
+ comp.toString(),
+ String.valueOf(encodeTime),
+ String.valueOf(decodeTime),
+ String.valueOf(data.size()),
+ String.valueOf(compressed_size),
+ String.valueOf(ratio)
+// f.toString(),
+// String.valueOf(index),
+// encoding.toString(),
+// comp.toString(),
+// String.valueOf(encodeTime),
+// String.valueOf(decodeTime),
+// String.valueOf(compressTime),
+// String.valueOf(uncompressTime),
+// String.valueOf(data.size()),
+// String.valueOf(compressed_size),
+// String.valueOf(ratio)
+ };
+ System.out.println(ratio);
+ writer.writeRecord(record);
+ }
+ }
+ break;
+ }
+ }
+ inputStream = Files.newInputStream(f.toPath());
+ loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
+ }
+ }
+ writer.close();
+ }
+ }
+
+}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVTest.java
deleted file mode 100644
index df560a1..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/TSDIFFBOSVTest.java
+++ /dev/null
@@ -1,1089 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-import com.csvreader.CsvReader;
-import com.csvreader.CsvWriter;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-
-import static java.lang.Math.pow;
-
-public class TSDIFFBOSVTest {
-
- public static long combine2Int(int int1, int int2) {
- return ((long) int1 << 32) | (int2 & 0xFFFFFFFFL);
- }
-
- public static int getTime(long long1) {
- return ((int) (long1 >> 32));
- }
-
- public static int getValue(long long1) {
- return ((int) (long1));
- }
-
- public static int getCount(long long1, int mask) {
- return ((int) (long1 & mask));
- }
- public static int getUniqueValue(long long1, int left_shift) {
- return ((int) ((long1) >> left_shift));
- }
-
- public static int getBitWith(int num) {
- if (num == 0) return 1;
- else return 32 - Integer.numberOfLeadingZeros(num);
- }
-
- public static void int2Bytes(int integer,int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
-
- public static void intByte2Bytes(int integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer);
- }
-
- private static void long2intBytes(long integer, int encode_pos , byte[] cur_byte) {
- cur_byte[encode_pos] = (byte) (integer >> 24);
- cur_byte[encode_pos+1] = (byte) (integer >> 16);
- cur_byte[encode_pos+2] = (byte) (integer >> 8);
- cur_byte[encode_pos+3] = (byte) (integer);
- }
-
- public static int bytes2Integer(byte[] encoded, int start, int num) {
- int value = 0;
- if (num > 4) {
- System.out.println("bytes2Integer error");
- return 0;
- }
- for (int i = 0; i < num; i++) {
- value <<= 8;
- int b = encoded[i + start] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- private static long bytesLong2Integer(byte[] encoded, int decode_pos) {
- long value = 0;
- for (int i = 0; i < 4; i++) {
- value <<= 8;
- int b = encoded[i + decode_pos] & 0xFF;
- value |= b;
- }
- return value;
- }
-
- public static void pack8Values(ArrayList<Integer> values, int offset, int width, int encode_pos, byte[] encoded_result) {
- int bufIdx = 0;
- int valueIdx = offset;
- // remaining bits for the current unfinished Integer
- int leftBit = 0;
-
- while (valueIdx < 8 + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
-
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values.get(valueIdx) << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
-
- while (leftSize >= width && valueIdx < 8 + offset) {
- // encode one Integer to the 'buffer'
- buffer |= (values.get(valueIdx)<< (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < 8 + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the
- // buffer
- buffer |= (values.get(valueIdx) >>> (width - leftSize));
- leftBit = width - leftSize;
- }
-
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- encoded_result[encode_pos] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- encode_pos ++;
- bufIdx++;
- if (bufIdx >= width) {
- return ;
- }
- }
- }
-// return encode_pos;
- }
-
- public static void unpack8Values(byte[] encoded, int offset,int width, ArrayList<Integer> result_list) {
- int byteIdx = offset;
- long buffer = 0;
- // total bits which have read from 'buf' to 'buffer'. i.e.,
- // number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
-
- while (valueIdx < 8) {
- // If current available bits are not enough to decode one Integer,
- // then add next byte from buf to 'buffer' until totalBits >= width
- while (totalBits < width) {
- buffer = (buffer << 8) | (encoded[byteIdx] & 0xFF);
- byteIdx++;
- totalBits += 8;
- }
-
- // If current available bits are enough to decode one Integer,
- // then decode one Integer one by one until left bits in 'buffer' is
- // not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- result_list.add ((int) (buffer >>> (totalBits - width)));
- valueIdx++;
- totalBits -= width;
- buffer = buffer & ((1L << totalBits) - 1);
- }
- }
- }
-
- public static int bitPacking(ArrayList<Integer> numbers, int start, int bit_width,int encode_pos, byte[] encoded_result) {
- int block_num = (numbers.size()-start) / 8;
- for(int i=0;i<block_num;i++){
- pack8Values( numbers, start+i*8, bit_width,encode_pos, encoded_result);
- encode_pos +=bit_width;
- }
-
- return encode_pos;
-
- }
-
- public static ArrayList<Integer> decodeBitPacking(
- byte[] encoded, int decode_pos, int bit_width, int block_size) {
- ArrayList<Integer> result_list = new ArrayList<>();
- int block_num = (block_size - 1) / 8;
-
- for (int i = 0; i < block_num; i++) { // bitpacking
- unpack8Values( encoded, decode_pos, bit_width, result_list);
- decode_pos += bit_width;
-
- }
- return result_list;
- }
-
-
- public static int[] getAbsDeltaTsBlock(
- int[] ts_block,
- int i,
- int block_size,
- int remaining,
- int[] min_delta) {
- int[] ts_block_delta = new int[remaining-1];
-
- int value_delta_min = Integer.MAX_VALUE;
- int value_delta_max = Integer.MIN_VALUE;
- int base = i*block_size+1;
- int end = i*block_size+remaining;
-
- int tmp_j_1 = ts_block[base-1];
- min_delta[0] =tmp_j_1;
- int j = base;
- int tmp_j;
-
- while(j<end){
- tmp_j = ts_block[j];
- int epsilon_v = tmp_j - tmp_j_1;
- ts_block_delta[j-base] = epsilon_v;
- if (epsilon_v < value_delta_min) {
- value_delta_min = epsilon_v;
- }
- if (epsilon_v > value_delta_max) {
- value_delta_max = epsilon_v;
- }
- tmp_j_1 = tmp_j;
- j++;
- }
- j = 0;
- end = remaining -1;
- while(j<end){
- ts_block_delta[j] = ts_block_delta[j] - value_delta_min;
- j++;
- }
-
- min_delta[1] = value_delta_min;
- min_delta[2] = (value_delta_max-value_delta_min);
-
-
- return ts_block_delta;
- }
-
-
- public static int encodeOutlier2Bytes(
- ArrayList<Integer> ts_block_delta,
- int bit_width,
- int encode_pos, byte[] encoded_result) {
-
- encode_pos = bitPacking(ts_block_delta, 0, bit_width, encode_pos, encoded_result);
-
- int n_k = ts_block_delta.size();
- int n_k_b = n_k / 8;
- long cur_remaining = 0; // encoded int
- int cur_number_bits = 0; // the bit width used of encoded int
- for (int i = n_k_b * 8; i < n_k; i++) {
- long cur_value = ts_block_delta.get(i);
- int cur_bit_width = bit_width; // remaining bit width of current value
-
- if (cur_number_bits + bit_width >= 32) {
- cur_remaining <<= (32 - cur_number_bits);
- cur_bit_width = bit_width - 32 + cur_number_bits;
- cur_remaining += ((cur_value >> cur_bit_width));
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- cur_remaining = 0;
- cur_number_bits = 0;
- }
-
- cur_remaining <<= cur_bit_width;
- cur_number_bits += cur_bit_width;
- cur_remaining += (((cur_value << (32 - cur_bit_width)) & 0xFFFFFFFFL) >> (32 - cur_bit_width)); //
- }
- cur_remaining <<= (32 - cur_number_bits);
- long2intBytes(cur_remaining,encode_pos,encoded_result);
- encode_pos += 4;
- return encode_pos;
-
- }
-
-
- public static ArrayList<Integer> decodeOutlier2Bytes(
- byte[] encoded,
- int decode_pos,
- int bit_width,
- int length,
- ArrayList<Integer> encoded_pos_result
- ) {
-
- int n_k_b = length / 8;
- int remaining = length - n_k_b * 8;
- ArrayList<Integer> result_list = new ArrayList<>(decodeBitPacking(encoded, decode_pos, bit_width, n_k_b * 8 + 1));
- decode_pos += n_k_b * bit_width;
-
- ArrayList<Long> int_remaining = new ArrayList<>();
- int int_remaining_size = remaining * bit_width / 32 + 1;
- for (int j = 0; j < int_remaining_size; j++) {
- int_remaining.add(bytesLong2Integer(encoded, decode_pos));
- decode_pos += 4;
- }
-
- int cur_remaining_bits = 32; // remaining bit width of current value
- long cur_number = int_remaining.get(0);
- int cur_number_i = 1;
- for (int i = n_k_b * 8; i < length; i++) {
- if (bit_width < cur_remaining_bits) {
- int tmp = (int) (cur_number >> (32 - bit_width));
- result_list.add(tmp);
- cur_number <<= bit_width;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits -= bit_width;
- } else {
- int tmp = (int) (cur_number >> (32 - cur_remaining_bits));
- int remain_bits = bit_width - cur_remaining_bits;
- tmp <<= remain_bits;
-
- cur_number = int_remaining.get(cur_number_i);
- cur_number_i++;
- tmp += (int) (cur_number >> (32 - remain_bits));
- result_list.add(tmp);
- cur_number <<= remain_bits;
- cur_number &= 0xFFFFFFFFL;
- cur_remaining_bits = 32 - remain_bits;
- }
- }
- encoded_pos_result.add(decode_pos);
- return result_list;
- }
-
- private static int BOSEncodeBits(int[] ts_block_delta,
- int final_k_start_value,
- int final_k_end_value,
- int max_delta_value,
- int[] min_delta,
- int encode_pos,
- byte[] cur_byte) {
- int block_size = ts_block_delta.length;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal = new ArrayList<>();
- int k1 = 0;
- int k2 = 0;
-
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
- int index_bitmap_outlier = 0;
- int cur_index_bitmap_outlier_bits = 0;
- for (int i = 0; i < block_size; i++) {
- int cur_value = ts_block_delta[i];
- if ( cur_value<= final_k_start_value) {
- final_left_outlier.add(cur_value);
- final_left_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 3;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 1;
- cur_index_bitmap_outlier_bits = 1;
- }
- k1++;
-
-
- } else if (cur_value >= final_k_end_value) {
- final_right_outlier.add(cur_value - final_k_end_value);
- final_right_outlier_index.add(i);
- if (cur_index_bitmap_outlier_bits % 8 != 7) {
- index_bitmap_outlier <<= 2;
- index_bitmap_outlier += 2;
- cur_index_bitmap_outlier_bits += 2;
- } else {
- index_bitmap_outlier <<= 1;
- index_bitmap_outlier += 1;
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- cur_index_bitmap_outlier_bits = 1;
- }
- k2++;
-
- } else {
- final_normal.add(cur_value - final_k_start_value-1);
- index_bitmap_outlier <<= 1;
- cur_index_bitmap_outlier_bits += 1;
- }
- if (cur_index_bitmap_outlier_bits % 8 == 0) {
- bitmap_outlier.add(index_bitmap_outlier);
- index_bitmap_outlier = 0;
- }
- }
- if (cur_index_bitmap_outlier_bits % 8 != 0) {
-
- index_bitmap_outlier <<= (8 - cur_index_bitmap_outlier_bits % 8);
-
- index_bitmap_outlier &= 0xFF;
- bitmap_outlier.add(index_bitmap_outlier);
- }
-
- int final_alpha = ((k1 + k2) * getBitWith(block_size-1)) <= (block_size + k1 + k2) ? 1 : 0;
-
-
- int k_byte = (k1 << 1);
- k_byte += final_alpha;
- k_byte += (k2 << 16);
-
-
- int2Bytes(k_byte,encode_pos,cur_byte);
- encode_pos += 4;
-
- int2Bytes(min_delta[0],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(min_delta[1],encode_pos,cur_byte);
- encode_pos += 4;
- int2Bytes(final_k_start_value,encode_pos,cur_byte);
- encode_pos += 4;
- int bit_width_final = getBitWith(final_k_end_value - final_k_start_value-2);
- intByte2Bytes(bit_width_final,encode_pos,cur_byte);
- encode_pos += 1;
- int left_bit_width = getBitWith(final_k_start_value);//final_left_max
- int right_bit_width = getBitWith(max_delta_value - final_k_end_value);//final_right_min
- intByte2Bytes(left_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- intByte2Bytes(right_bit_width,encode_pos,cur_byte);
- encode_pos += 1;
- if (final_alpha == 0) { // 0
-
- for (int i : bitmap_outlier) {
-
- intByte2Bytes(i,encode_pos,cur_byte);
- encode_pos += 1;
- }
- } else {
- encode_pos = encodeOutlier2Bytes(final_left_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- encode_pos = encodeOutlier2Bytes(final_right_outlier_index, getBitWith(block_size-1),encode_pos,cur_byte);
- }
-// if(k1+k2!=block_size)
- encode_pos = encodeOutlier2Bytes(final_normal, bit_width_final,encode_pos,cur_byte);
- if (k1 != 0)
- encode_pos = encodeOutlier2Bytes(final_left_outlier, left_bit_width,encode_pos,cur_byte);
- if (k2 != 0)
- encode_pos = encodeOutlier2Bytes(final_right_outlier, right_bit_width,encode_pos,cur_byte);
- return encode_pos;
-
- }
-
-
- private static int BOSBlockEncoder(int[] ts_block, int block_i, int block_size, int remaining ,int encode_pos , byte[] cur_byte) {
-
- int[] min_delta = new int[3];
- int[] ts_block_delta = getAbsDeltaTsBlock(ts_block, block_i, block_size, remaining, min_delta);
-
-
- block_size = remaining-1;
- int max_delta_value = min_delta[2];
- int[] value_list = new int[block_size];
- int unique_value_count = 0;
- int[] value_count_list = new int[max_delta_value+1];
- for(int value:ts_block_delta){
- if(value_count_list[value]==0){
- value_count_list[value] = 1;
- value_list[unique_value_count] = value;
- unique_value_count ++;
- }else{
- value_count_list[value] ++;
- }
- }
-
- int left_shift = getBitWith(block_size);
- int mask = (1 << left_shift) - 1;
- long[] sorted_value_list = new long[unique_value_count];
- int count = 0;
-
- for(int i=0;i<unique_value_count;i++){
- int value = value_list[i];
- sorted_value_list[i] = (((long) value) << left_shift) + value_count_list[value];
- }
- Arrays.sort(sorted_value_list);
-
- for(int i=0;i<unique_value_count;i++){
- count += getCount(sorted_value_list[i], mask);
- sorted_value_list[i] = (((long)getUniqueValue(sorted_value_list[i], left_shift) ) << left_shift) + count;//new_value_list[i]
- }
-
-
- int final_k_start_value = -1;
- int final_k_end_value = max_delta_value+1;
-
- int min_bits = 0;
- min_bits += (getBitWith(final_k_end_value - final_k_start_value ) * (block_size));
-
- for (int end_value_i = 1; end_value_i < unique_value_count; end_value_i++) {
-
- int k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
-
- int cur_bits = 0;
- int cur_k2;
- int cur_k1 = 0;
- int k_start_value = -1;
- cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
- cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
- if (cur_k1 + cur_k2 != block_size)
- cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(k_end_value- k_start_value -2);
- if (cur_k2 != 0)
- cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value + 1);//min_upper_outlier
-
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_end_value = k_end_value;
- }
- }
-
-
- int start_value_size = unique_value_count;
- for (int start_value_i = 0; start_value_i < start_value_size; start_value_i++) {
- int k_start_value = getUniqueValue(sorted_value_list[start_value_i], left_shift) ;
-
- int cur_k1 = getCount(sorted_value_list[start_value_i],mask);
-
- int k_end_value;
- int cur_bits;
- int cur_k2;
- k_end_value = max_delta_value + 1;
-
- cur_bits = 0;
- cur_k2 = 0;
- cur_bits += Math.min((cur_k2 + cur_k1) * getBitWith(block_size-1), block_size + cur_k2 + cur_k1);
- if (cur_k1 != 0)
- cur_bits += cur_k1 * getBitWith(k_start_value + 1);
- if (cur_k1 + cur_k2 != block_size)
- cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(k_end_value- k_start_value -2);
-
-
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value;
- final_k_end_value = k_end_value;
- }
-
- for (int end_value_i = start_value_i + 1; end_value_i < unique_value_count; end_value_i++) {
-
- k_end_value = getUniqueValue(sorted_value_list[end_value_i], left_shift);
-
- cur_bits = 0;
- cur_k2 = block_size - getCount(sorted_value_list[end_value_i-1],mask);
-
- cur_bits += Math.min((cur_k1 + cur_k2) * getBitWith(block_size-1), block_size + cur_k1 + cur_k2);
- if (cur_k1 != 0)
- cur_bits += cur_k1 * getBitWith(k_start_value);
- if (cur_k1 + cur_k2 != block_size)
- cur_bits += (block_size - cur_k1 - cur_k2) * getBitWith(k_end_value - k_start_value-2);
- if (cur_k2 != 0)
- cur_bits += cur_k2 * getBitWith(max_delta_value - k_end_value);
-
-
- if (cur_bits < min_bits) {
- min_bits = cur_bits;
- final_k_start_value = k_start_value;
- final_k_end_value = k_end_value;
- }
-
- }
- }
-
-
- encode_pos = BOSEncodeBits(ts_block_delta, final_k_start_value, final_k_end_value, max_delta_value,
- min_delta, encode_pos , cur_byte);
-
- return encode_pos;
- }
-
-
- public static int BOSEncoder(
- int[] data, int block_size, byte[] encoded_result) {
- block_size++;
-
- int length_all = data.length;
-
- int encode_pos = 0;
- int2Bytes(length_all,encode_pos,encoded_result);
- encode_pos += 4;
-
- int block_num = length_all / block_size;
- int2Bytes(block_size,encode_pos,encoded_result);
- encode_pos+= 4;
-
- for (int i = 0; i < block_num; i++) {
- encode_pos = BOSBlockEncoder(data, i, block_size, block_size,encode_pos,encoded_result);
- }
-
- int remaining_length = length_all - block_num * block_size;
- if (remaining_length <= 3) {
- for (int i = remaining_length; i > 0; i--) {
- int2Bytes(data[data.length - i], encode_pos, encoded_result);
- encode_pos += 4;
- }
-
- }
- else {
-
- int start = block_num * block_size;
- int remaining = length_all-start;
- encode_pos = BOSBlockEncoder(data, block_num, block_size,remaining, encode_pos,encoded_result);
-
- }
-
-
- return encode_pos;
- }
-
- public static int BOSBlockDecoder(byte[] encoded, int decode_pos, int[] value_list, int block_size, int[] value_pos_arr) {
-
-
- int k_byte = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int k1_byte = (int) (k_byte % pow(2, 16));
- int k1 = k1_byte / 2;
- int final_alpha = k1_byte % 2;
-
- int k2 = (int) (k_byte / pow(2, 16));
-
- int value0 = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] =value0;
- value_pos_arr[0] ++;
-
- int min_delta = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int final_k_start_value = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
- int bit_width_final = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- int left_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
- int right_bit_width = bytes2Integer(encoded, decode_pos, 1);
- decode_pos += 1;
-
- ArrayList<Integer> final_left_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_right_outlier_index = new ArrayList<>();
- ArrayList<Integer> final_left_outlier = new ArrayList<>();
- ArrayList<Integer> final_right_outlier = new ArrayList<>();
- ArrayList<Integer> final_normal= new ArrayList<>();;
- ArrayList<Integer> bitmap_outlier = new ArrayList<>();
-
-
- if (final_alpha == 0) {
- int bitmap_bytes = (int) Math.ceil((double) (block_size + k1 + k2) / (double) 8);
- for (int i = 0; i < bitmap_bytes; i++) {
- bitmap_outlier.add(bytes2Integer(encoded, decode_pos, 1));
- decode_pos += 1;
- }
- int bitmap_outlier_i = 0;
- int remaining_bits = 8;
- int tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- int i = 0;
- while (i < block_size ) {
- if (remaining_bits > 1) {
- int bit_i = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- if (remaining_bits == 0) {
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- }
- } else if (remaining_bits == 1) {
- int bit_i = tmp & 0x1;
- remaining_bits = 8;
- if (bitmap_outlier_i >= bitmap_bytes) break;
- tmp = bitmap_outlier.get(bitmap_outlier_i);
- bitmap_outlier_i++;
- if (bit_i == 1) {
- int bit_left_right = (tmp >> (remaining_bits - 1)) & 0x1;
- remaining_bits -= 1;
- if (bit_left_right == 1) {
- final_left_outlier_index.add(i);
- } else {
- final_right_outlier_index.add(i);
- }
- }
- }
- i++;
- }
- } else {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k1, decode_pos_result_left);
- decode_pos = (decode_pos_result_left.get(0));
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier_index = decodeOutlier2Bytes(encoded, decode_pos, getBitWith(block_size-1), k2, decode_pos_result_right);
- decode_pos = (decode_pos_result_right.get(0));
- }
-
-
- ArrayList<Integer> decode_pos_normal = new ArrayList<>();
-// if(k1+k2!=block_size)
- final_normal = decodeOutlier2Bytes(encoded, decode_pos, bit_width_final, block_size - k1 - k2, decode_pos_normal);
-
- decode_pos = decode_pos_normal.get(0);
- if (k1 != 0) {
- ArrayList<Integer> decode_pos_result_left = new ArrayList<>();
- final_left_outlier = decodeOutlier2Bytes(encoded, decode_pos, left_bit_width, k1, decode_pos_result_left);
- decode_pos = decode_pos_result_left.get(0);
- }
- if (k2 != 0) {
- ArrayList<Integer> decode_pos_result_right = new ArrayList<>();
- final_right_outlier = decodeOutlier2Bytes(encoded, decode_pos, right_bit_width, k2, decode_pos_result_right);
- decode_pos = decode_pos_result_right.get(0);
- }
- int left_outlier_i = 0;
- int right_outlier_i = 0;
- int normal_i = 0;
- int pre_v = value0;
- int final_k_end_value = (int) (final_k_start_value + pow(2, bit_width_final));
-
-
- for (int i = 0; i < block_size; i++) {
- int current_delta;
- if (left_outlier_i >= k1) {
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- } else if (i == final_left_outlier_index.get(left_outlier_i)) {
- current_delta = min_delta + final_left_outlier.get(left_outlier_i);
- left_outlier_i++;
- } else {
-
- if (right_outlier_i >= k2) {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- } else if (i == final_right_outlier_index.get(right_outlier_i)) {
- current_delta = min_delta + final_right_outlier.get(right_outlier_i) + final_k_end_value;
- right_outlier_i++;
- } else {
- current_delta = min_delta + final_normal.get(normal_i) + final_k_start_value;
- normal_i++;
- }
- }
-
- pre_v = current_delta + pre_v;
- value_list[value_pos_arr[0]] = pre_v;
- value_pos_arr[0]++;
- }
- return decode_pos;
- }
-
- public static void BOSDecoder(byte[] encoded) {
-
- int decode_pos = 0;
- int length_all = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- int block_size = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
-
-
-
- int block_num = length_all / block_size;
- int remain_length = length_all - block_num * block_size;
-
-
- int[] value_list = new int[length_all+block_size];
- block_size--;
-
- int[] value_pos_arr = new int[1];
- for (int k = 0; k < block_num; k++) {
-
-
- decode_pos = BOSBlockDecoder(encoded, decode_pos, value_list, block_size,value_pos_arr);
-
- }
-
- if (remain_length <= 3) {
- for (int i = 0; i < remain_length; i++) {
- int value_end = bytes2Integer(encoded, decode_pos, 4);
- decode_pos += 4;
- value_list[value_pos_arr[0]] = value_end;
- value_pos_arr[0]++;
- }
- } else {
- remain_length --;
- BOSBlockDecoder(encoded, decode_pos, value_list, remain_length, value_pos_arr);
- }
- }
-
- @Test
- public void BOSOptimalTest() throws IOException {
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/"; // your data path
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
- String output_parent_dir = parent_dir + "vldb/compression_ratio/bos";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- ArrayList<Integer> dataset_block_size = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
- dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- dataset_block_size.add(1024);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-// dataset_block_size.add(1024);
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-// dataset_block_size.add(2048);
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-// dataset_block_size.add(1024);
-
- int repeatTime2 = 100;
- for (int file_i = 8; file_i < 9; file_i++) {
-//
-// for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
-// f=tempList[2];
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
- while (loader.readRecord()) {
-// String value = loader.getValues()[index];
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
-// data.add(Integer.valueOf(value));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- int length = 0;
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- length = BOSEncoder(data2_arr, dataset_block_size.get(file_i), encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- compressed_size += length;
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
-
- String[] record = {
- f.toString(),
- "TS_2DIFF+BOS-O",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-// break;
- }
- writer.close();
- }
- }
-
- @Test
- public void BOSVaryBlockSize() throws IOException {
- String parent_dir = "/Users/zihanguo/Downloads/R/outlier/outliier_code/encoding-outlier/";
-// String parent_dir = "/Users/xiaojinzhao/Desktop/encoding-outlier/"; // your data path
- String output_parent_dir = parent_dir + "vldb/compression_ratio/block_size_bos";
- String input_parent_dir = parent_dir + "trans_data/";
- ArrayList<String> input_path_list = new ArrayList<>();
- ArrayList<String> output_path_list = new ArrayList<>();
- ArrayList<String> dataset_name = new ArrayList<>();
- dataset_name.add("CS-Sensors");
- dataset_name.add("Metro-Traffic");
- dataset_name.add("USGS-Earthquakes");
- dataset_name.add("YZ-Electricity");
- dataset_name.add("GW-Magnetic");
- dataset_name.add("TY-Fuel");
- dataset_name.add("Cyber-Vehicle");
- dataset_name.add("Vehicle-Charge");
-// dataset_name.add("Nifty-Stocks");
- dataset_name.add("TH-Climate");
- dataset_name.add("TY-Transport");
- dataset_name.add("EPM-Education");
-
- for (String value : dataset_name) {
- input_path_list.add(input_parent_dir + value);
- }
-
- output_path_list.add(output_parent_dir + "/CS-Sensors_ratio.csv"); // 0
- output_path_list.add(output_parent_dir + "/Metro-Traffic_ratio.csv");// 1
- output_path_list.add(output_parent_dir + "/USGS-Earthquakes_ratio.csv");// 2
-
- output_path_list.add(output_parent_dir + "/YZ-Electricity_ratio.csv"); // 3
-
- output_path_list.add(output_parent_dir + "/GW-Magnetic_ratio.csv"); //4
-
- output_path_list.add(output_parent_dir + "/TY-Fuel_ratio.csv");//5
-
- output_path_list.add(output_parent_dir + "/Cyber-Vehicle_ratio.csv"); //6
-
- output_path_list.add(output_parent_dir + "/Vehicle-Charge_ratio.csv");//7
-
- output_path_list.add(output_parent_dir + "/Nifty-Stocks_ratio.csv");//8
-
- output_path_list.add(output_parent_dir + "/TH-Climate_ratio.csv");//9
-
- output_path_list.add(output_parent_dir + "/TY-Transport_ratio.csv");//10
-
- output_path_list.add(output_parent_dir + "/EPM-Education_ratio.csv");//11
-
- int repeatTime2 = 100;
-// for (int file_i = 9; file_i < 10; file_i++) {
- for (int file_i = 0; file_i < input_path_list.size(); file_i++) {
-
- String inputPath = input_path_list.get(file_i);
- System.out.println(inputPath);
- String Output = output_path_list.get(file_i);
-
-
- File file = new File(inputPath);
- File[] tempList = file.listFiles();
-
- CsvWriter writer = new CsvWriter(Output, ',', StandardCharsets.UTF_8);
-
- String[] head = {
- "Input Direction",
- "Encoding Algorithm",
- "Encoding Time",
- "Decoding Time",
- "Points",
- "Compressed Size",
- "Block Size",
- "Compression Ratio"
- };
- writer.writeRecord(head); // write header to output file
-
- assert tempList != null;
-
- for (File f : tempList) {
- System.out.println(f);
- InputStream inputStream = Files.newInputStream(f.toPath());
-
- CsvReader loader = new CsvReader(inputStream, StandardCharsets.UTF_8);
- ArrayList<Integer> data1 = new ArrayList<>();
- ArrayList<Integer> data2 = new ArrayList<>();
-
-
- loader.readHeaders();
-
- while (loader.readRecord()) {
- data1.add(Integer.valueOf(loader.getValues()[0]));
- data2.add(Integer.valueOf(loader.getValues()[1]));
- }
- inputStream.close();
- int[] data2_arr = new int[data1.size()];
- for(int i = 0;i<data2.size();i++){
- data2_arr[i] = data2.get(i);
- }
- byte[] encoded_result = new byte[data2_arr.length*4];
-
-// for (int block_size_i = 6; block_size_i > 5; block_size_i--) {
- for (int block_size_i = 13; block_size_i > 4; block_size_i--) {
- int block_size = (int) Math.pow(2, block_size_i);
- System.out.println(block_size);
- long encodeTime = 0;
- long decodeTime = 0;
- double ratio = 0;
- double compressed_size = 0;
-
-
- long s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++) {
- compressed_size = BOSEncoder(data2_arr, block_size, encoded_result);
- }
-
- long e = System.nanoTime();
- encodeTime += ((e - s) / repeatTime2);
- double ratioTmp = compressed_size / (double) (data1.size() * Integer.BYTES);
- ratio += ratioTmp;
- s = System.nanoTime();
- for (int repeat = 0; repeat < repeatTime2; repeat++)
- BOSDecoder(encoded_result);
- e = System.nanoTime();
- decodeTime += ((e - s) / repeatTime2);
-
- String[] record = {
- f.toString(),
- "TS_2DIFF+BOS-V",
- String.valueOf(encodeTime),
- String.valueOf(decodeTime),
- String.valueOf(data1.size()),
- String.valueOf(compressed_size),
- String.valueOf(block_size_i),
- String.valueOf(ratio)
- };
- writer.writeRecord(record);
- System.out.println(ratio);
-
- }
-
- }
- writer.close();
-
- }
- }
-
-
-
-}
diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/myPFOR.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/myPFOR.java
deleted file mode 100644
index 03b33ab..0000000
--- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/encoding/myPFOR.java
+++ /dev/null
@@ -1,333 +0,0 @@
-package org.apache.iotdb.tsfile.encoding;
-
-
-import com.kamikaze.pfordelta.Simple16;
-import org.jetbrains.annotations.NotNull;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Random;
-
-public class myPFOR {
- // NOTE: we expect the blockSize is always < (1<<(31-POSSIBLE_B_BITS)).
- // For example, in the current default settings,
- // the blockSize < (1<<(31-5)), that is, < 2^27, the commonly used block
- // size is 128 or 256.
-
- // All possible values of b in the PForDelta algorithm
- private static final int[] POSSIBLE_B = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
- 10, 11, 12, 13, 16, 20, 28};
- private static final int MAX_BITS = 32;
-
- private static final int HEADER_NUM = 4;
- private static final int HEADER_SIZE = MAX_BITS * HEADER_NUM;
- private static int min = Integer.MAX_VALUE;
-
- public static int[] compressOneBlockOpt(final int[] inBlock,
- int blockSize) {
- // find the best b that may lead to the smallest overall
- // compressed size
- int[] inBlock2 = new int[blockSize];
- min = Integer.MAX_VALUE;
- for (int k : inBlock) {
- if (k < min) {
- min = k;
- }
- }
- for (int j = 0; j < inBlock.length; j++){
- inBlock2[j] = inBlock[j] - min;
- }
- int currentB = POSSIBLE_B[0];
- int[] outBlock = null;
- int tmpB = currentB;
- int optSize = estimateCompressedSize(inBlock2, tmpB,
- blockSize);
- for (int i = 1; i < POSSIBLE_B.length; ++i) {
- tmpB = POSSIBLE_B[i];
- int curSize = estimateCompressedSize(inBlock2,
- tmpB, blockSize);
- if (curSize < optSize) {
- currentB = tmpB;
- optSize = curSize;
- }
- }
- // 使用相同的b分block压缩
- outBlock = compressOneBlock(inBlock2, currentB, blockSize);
-
- return outBlock;
- }
-
- public static int decompressOneBlock(int[] outBlock, final int[] inBlock,
- int blockSize) {
- int[] expAux = new int[blockSize];
-
- int expNum = inBlock[1];
- int bits = inBlock[0];
- int index_first_exp = inBlock[2];
- min = inBlock[3];
-
- int offset = HEADER_SIZE;
- int compressedBits = 0;
- if (bits == 0) {
- Arrays.fill(outBlock, 0);
- } else {
-
- compressedBits = decompressBBitSlots(outBlock, inBlock,
- blockSize, bits);
- }
- offset += compressedBits;
-
-
- if (expNum > 0) {
- compressedBits = decompressBlockByS16(expAux, inBlock,
- offset, expNum);
- offset += compressedBits;
-
- int exp_index = index_first_exp;
- int shift;
- for (int i = 0; i < expNum; i++) {
- int exp_value = expAux[i];
- shift = outBlock[exp_index];
- outBlock[exp_index] = exp_value;
- exp_index += (shift + 1);
- }
-
- }
-
- for (int i = 0; i < blockSize; i++){
- outBlock[i] += min;
- }
- return offset;
- }
-
- public static int estimateCompressedSize(int[] inputBlock, int bits,
- int blockSize) {
- int maxNoExp = (1 << bits) - 1;
- // Size of the header and the bits-bit slots
- int outputOffset = HEADER_SIZE + bits * blockSize;
- int expNum = 0;
-
- for (int i = 0; i < blockSize; ++i) {
- if (inputBlock[i] > maxNoExp) {
- expNum++;
- }
- }
- outputOffset += (expNum << 5);
-
- return outputOffset;
- }
-
- public static int[] compressOneBlock(int[] inputBlock, int bits,
- int blockSize) {
- int maxCompBitSize = HEADER_SIZE + blockSize
- * (MAX_BITS + MAX_BITS + MAX_BITS) + 32;
-
- int[] tmpCompressedBlock = new int[(maxCompBitSize >>> 5)];
-
- int outputOffset = HEADER_SIZE;
- int expUpperBound = 1 << bits;
- int expNum = 0;
- int expNum_more = 0;
- ArrayList<Integer> exp_index = new ArrayList<>();
- ArrayList<Integer> exp_index_more = new ArrayList<>();
- ArrayList<Integer> shift = new ArrayList<>();
- ArrayList<Integer> exp_value = new ArrayList<>();
- ArrayList<Integer> exp_value_more = new ArrayList<>();
- for (int i = 0; i < inputBlock.length; i++) {
- if (inputBlock[i] >= expUpperBound) {
- expNum++;
- exp_value.add(inputBlock[i]);
- exp_index.add(i);
- }
- }
-
- if (expNum == 0){
-
- for (int i = 0; i < blockSize; i++){
- writeBits(tmpCompressedBlock, inputBlock[i], outputOffset, bits);
- outputOffset += bits;
- }
- } else if (expNum == 1) {
-
- for (int i = 0; i < blockSize; i++){
- if (i != exp_index.get(0)){
- writeBits(tmpCompressedBlock, inputBlock[i], outputOffset, bits);
- }else {
- writeBits(tmpCompressedBlock, 0, outputOffset, bits);
- }
- outputOffset += bits;
- }
- }else {
-
- exp_index_more.add(exp_index.get(0));
- exp_value_more.add(exp_value.get(0));
- expNum_more++;
- for (int i = 0; i < expNum - 1; i++){
- if (exp_index.get(i + 1) - exp_index.get(i) <= (1 << bits)){
-
- exp_index_more.add(exp_index.get(i + 1));
- exp_value_more.add(exp_value.get(i + 1));
- expNum_more++;
- shift.add(exp_index.get(i + 1) - exp_index.get(i) - 1);
- }else {
-
- int tag = exp_index.get(i);
- while (exp_index.get(i + 1) - tag > (1 << bits)){
-
- tag += (1 << bits);
- expNum_more++;
- exp_index_more.add(tag);
- exp_value_more.add(inputBlock[tag]);
- shift.add((1 << bits) - 1);
- }
-
- exp_index_more.add(exp_index.get(i + 1));
- exp_value_more.add(exp_value.get(i + 1));
- expNum_more++;
- shift.add(exp_index.get(i + 1) - tag - 1);
- }
- }
- shift.add(0);
-
- int j = 0;
- for (int i = 0; i < blockSize; i++){
- if (j < expNum_more && exp_index_more.get(j) == i){
-
- writeBits(tmpCompressedBlock, shift.get(j), outputOffset, bits);
- j++;
- }else {
-
- writeBits(tmpCompressedBlock, inputBlock[i], outputOffset, bits);
- }
- outputOffset += bits;
- }
- }
-
- tmpCompressedBlock[0] = bits;
- tmpCompressedBlock[1] = expNum;
- if (tmpCompressedBlock[1] > 1){
- tmpCompressedBlock[1] = expNum_more;
- }
- if (expNum > 0){
- tmpCompressedBlock[2] = exp_index.get(0);
- }else{
- tmpCompressedBlock[2] = 0;
- }
- tmpCompressedBlock[3] = min;
-
- if (expNum == 1){
- int[] expAux = new int[1];
- expAux[0] = exp_value.get(0);
- int compressedBitSize = compressBlockByS16(
- tmpCompressedBlock, outputOffset, expAux,
- expNum);
- outputOffset += compressedBitSize;
- }
- else if (expNum > 1) {
-
- int[] expAux = new int[expNum_more];
- for (int i = 0; i < expNum_more; i++){
- expAux[i] = exp_value_more.get(i);
- }
- int compressedBitSize = compressBlockByS16(
- tmpCompressedBlock, outputOffset, expAux,
- expNum_more);
- outputOffset += compressedBitSize;
- }
-
- int compressedSizeInInts = (outputOffset + 31) >>> 5;
- int[] compBlock;
- compBlock = new int[compressedSizeInInts];
- System.arraycopy(tmpCompressedBlock, 0, compBlock, 0,
- compressedSizeInInts);
-
- return compBlock;
- }
- public static int decompressBBitSlots(int[] outDecompSlots,
- int[] inCompBlock, int blockSize, int bits) {
- int compressedBitSize = 0;
- int offset = HEADER_SIZE;
- for (int i = 0; i < blockSize; i++) {
- outDecompSlots[i] = readBits(inCompBlock, offset, bits);
- offset += bits;
- }
- compressedBitSize = bits * blockSize;
-
- return compressedBitSize;
- }
-
- private static int compressBlockByS16(int[] outCompBlock,
- int outStartOffsetInBits, int[] inBlock, int blockSize) {
- int outOffset = (outStartOffsetInBits + 31) >>> 5;
- int num, inOffset = 0, numLeft;
- for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
- num = Simple16.s16Compress(outCompBlock, outOffset,
- inBlock, inOffset, numLeft, blockSize);
- outOffset++;
- inOffset += num;
- }
- int compressedBitSize = (outOffset << 5) - outStartOffsetInBits;
- return compressedBitSize;
- }
-
- public static int decompressBlockByS16(int[] outDecompBlock,
- int[] inCompBlock, int inStartOffsetInBits, int blockSize) {
- int inOffset = (inStartOffsetInBits + 31) >>> 5;
- int num, outOffset = 0, numLeft;
- for (numLeft = blockSize; numLeft > 0; numLeft -= num) {
- num = Simple16.s16Decompress(outDecompBlock, outOffset,
- inCompBlock, inOffset, numLeft);
- outOffset += num;
- inOffset++;
- }
- int compressedBitSize = (inOffset << 5) - inStartOffsetInBits;
- return compressedBitSize;
- }
-
- public static void writeBits(int[] out, int val, int outOffset,
- int bits) {
- if (bits == 0)
- return;
- final int index = outOffset >>> 5;
- final int skip = outOffset & 0x1f;
- val &= (0xffffffff >>> (32 - bits));
- out[index] |= (val << skip);
- if (32 - skip < bits) {
- out[index + 1] |= (val >>> (32 - skip));
- }
- }
-
- public static int readBits(int[] in, final int inOffset,
- final int bits) {
- final int index = inOffset >>> 5;
- final int skip = inOffset & 0x1f;
- int val = in[index] >>> skip;
- if (32 - skip < bits) {
- val |= (in[index + 1] << (32 - skip));
- }
- return val & (0xffffffff >>> (32 - bits));
- }
-
- public static void main(@NotNull String[] args) throws IOException {
- int length = 75809;
- int width = 10000000;
- int[] timeSeries = new int[length];
- Random rand = new Random();
-
- for (int i = 0; i < length; i++) {
- timeSeries[i] = rand.nextInt(width) - width/2;
- }
- int[] outBlock;
- outBlock = compressOneBlockOpt(timeSeries,length);
- int[] uncompressed = new int[length];
- int size = decompressOneBlock(uncompressed,outBlock,length);
- System.out.println(size);
- System.out.println(uncompressed.length);
- for (int i = 0; i < length; i++){
- if (timeSeries[i] != uncompressed[i]){
- System.out.println(i);
- }
- }
- }
-}
