add FLEA (#15773)
diff --git a/client-cpp/src/main/Session.h b/client-cpp/src/main/Session.h index 7d10356..c2de083 100644 --- a/client-cpp/src/main/Session.h +++ b/client-cpp/src/main/Session.h
@@ -217,7 +217,7 @@ GORILLA = (char)8, ZIGZAG = (char)9, FREQ = (char)10, - PERIOD = (char)14 + FLEA = (char)14 }; }
diff --git a/client-py/iotdb/utils/IoTDBConstants.py b/client-py/iotdb/utils/IoTDBConstants.py index eda947e..00c4d8b 100644 --- a/client-py/iotdb/utils/IoTDBConstants.py +++ b/client-py/iotdb/utils/IoTDBConstants.py
@@ -61,7 +61,7 @@ GORILLA = 8 ZIGZAG = 9 FREQ = 10 - PERIOD = 14 + FLEA = 14 # this method is implemented to avoid the issue reported by: # https://bugs.python.org/issue30545
diff --git a/docs/UserGuide/Data-Concept/Encoding.md b/docs/UserGuide/Data-Concept/Encoding.md index 6fe412f..4771b1c 100644 --- a/docs/UserGuide/Data-Concept/Encoding.md +++ b/docs/UserGuide/Data-Concept/Encoding.md
@@ -64,11 +64,9 @@ ZIGZAG encoding maps signed integers to unsigned integers so that numbers with a small absolute value (for instance, -1) have a small variant encoded value too. It does this in a way that "zig-zags" back and forth through the positive and negative integers. -* PERIOD +* FLEA -PERIOD encoding is suitable for encoding data that is periodic in nature. By calculating the periodicity of the data, it is possible to use lossy frequency-domain data to preserve the periodic character of the data, thus applying a smaller space for storing the residual data. - -For the residual data, since most of the residual data is small and a small portion of the residual data is large, PERIOD encoding employs a separate storage method, storing the high and low bits of the residual data separately. +FLEA (Frequency-based Lossless Encoding Algorithm) is a novel lossless compression algorithm designed specifically for patterned time series. It uniquely leverages frequency-domain analysis within a rate-optimal framework, decomposing data into frequency and residual components whose encoding costs are jointly minimized by adaptively selecting a quantization parameter. By further employing a bi-regional encoding strategy with specialized coders for dense and sparse frequency data, alongside a hybrid residual encoder, FLEA achieves state-of-the-art compression ratios, particularly on data exhibiting periodic or cyclical structures. ## Correspondence between data type and encoding
diff --git a/iotdb-client/client-cpp/src/main/Session.h b/iotdb-client/client-cpp/src/main/Session.h index 24b2622..6ab03d8 100644 --- a/iotdb-client/client-cpp/src/main/Session.h +++ b/iotdb-client/client-cpp/src/main/Session.h
@@ -231,7 +231,7 @@ CHIMP = (char)11, SPRINTZ = (char)12, RLBE = (char)13, - PERIOD = (char)14 + FLEA = (char)14 }; }
diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py index 3dfbe3c..85ceeed 100644 --- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py +++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
@@ -54,7 +54,7 @@ CHIMP = 11 SPRINTZ = 12 RLBE = 13 - PERIOD = 14 + FLEA = 14 @unique
diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 6f35f51..5aa715a 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java
@@ -238,7 +238,7 @@ "ON", "OFF", "OF", - "PERIOD", + "FLEA", "PROCESSLIST", "PREVIOUS", "PREVIOUSUNTILLAST",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 714689a..26a2a03 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -59,7 +59,7 @@ intSet.add(TSEncoding.CHIMP); intSet.add(TSEncoding.SPRINTZ); intSet.add(TSEncoding.RLBE); - intSet.add(TSEncoding.PERIOD); + intSet.add(TSEncoding.FLEA); schemaChecker.put(TSDataType.INT32, intSet); schemaChecker.put(TSDataType.INT64, intSet);
diff --git a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java index 3df1318..c4044a3 100644 --- a/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java +++ b/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBDatabaseMetadata.java
@@ -66,8 +66,8 @@ private static final String METHOD_NOT_SUPPORTED_STRING = "Method not supported"; // when running the program in IDE, we can not get the version info using // getImplementationVersion() - private static final String DATABASE_VERSION = - IoTDBDatabaseMetadata.class.getPackage().getImplementationVersion() != null + private static final String DATABASE_VERSION = IoTDBDatabaseMetadata.class.getPackage() + .getImplementationVersion() != null ? IoTDBDatabaseMetadata.class.getPackage().getImplementationVersion() : "UNKNOWN"; private long sessionId; @@ -84,382 +84,382 @@ static { String[] allIotdbSQLKeywords = { - "ALTER", - "ADD", - "ALIAS", - "ALL", - "AVG", - "ALIGN", - "ATTRIBUTES", - "AS", - "ASC", - "BY", - "BOOLEAN", - "BITMAP", - "CREATE", - "CONFIGURATION", - "COMPRESSOR", - "CHILD", - "COUNT", - "COMPRESSION", - "CLEAR", - "CACHE", - "CONTAIN", - "CONCAT", - "DELETE", - "DEVICE", - "DESCRIBE", - "DATATYPE", - "DOUBLE", - "DIFF", - "DROP", - "DEVICES", - "DISABLE", - "DESC", - "ENCODING", - "FROM", - "FILL", - "FLOAT", - "FLUSH", - "FIRST_VALUE", - "FULL", - "FALSE", - "FOR", - "FUNCTION", - "FUNCTIONS", - "GRANT", - "GROUP", - "GORILLA", - "GLOBAL", - "GZIP", - "INSERT", - "INTO", - "INT32", - "INT64", - "INDEX", - "INFO", - "KILL", - "LIMIT", - "LINEAR", - "LABEL", - "LINK", - "LIST", - "LOAD", - "LEVEL", - "LAST_VALUE", - "LAST", - "LZO", - "LZ4", - "LATEST", - "LIKE", - "METADATA", - "MERGE", - "MOVE", - "MIN_TIME", - "MAX_TIME", - "MIN_VALUE", - "MAX_VALUE", - "NOW", - "NODES", - "ORDER", - "OFFSET", - "ON", - "OFF", - "OF", - "PERIOD", - "PROCESSLIST", - "PREVIOUS", - "PREVIOUSUNTILLAST", - "PROPERTY", - "PLAIN", - "PLAIN_DICTIONARY", - "PRIVILEGES", - "PASSWORD", - "PATHS", - "PAA", - "PLA", - "PARTITION", - "QUERY", - "ROOT", - "RLE", - "REGULAR", - "ROLE", - "REVOKE", - "REMOVE", - "RENAME", - "SELECT", - "SHOW", - "SET", - "SLIMIT", - "SOFFSET", - "STORAGE", - "SUM", - "SNAPPY", - "SNAPSHOT", - "SCHEMA", - "TO", - "TIMESERIES", - "TIMESTAMP", - "TEXT", - "TS_2DIFF", - "TRACING", - "TTL", - "TASK", - "TIME", - "TAGS", - "TRUE", - "TEMPORARY", - "TOP", - "TOLERANCE", - "UPDATE", - "UNLINK", - "UPSERT", - "USING", - "USER", - "UNSET", - "UNCOMPRESSED", - "VALUES", - "VERSION", - "WHERE", - "WITH", - "WATERMARK_EMBEDDING" + "ALTER", + "ADD", + "ALIAS", + "ALL", + "AVG", + "ALIGN", + "ATTRIBUTES", + "AS", + "ASC", + "BY", + "BOOLEAN", + "BITMAP", + "CREATE", + "CONFIGURATION", + "COMPRESSOR", + "CHILD", + "COUNT", + "COMPRESSION", + "CLEAR", + "CACHE", + "CONTAIN", + "CONCAT", + "DELETE", + "DEVICE", + "DESCRIBE", + "DATATYPE", + "DOUBLE", + "DIFF", + "DROP", + "DEVICES", + "DISABLE", + "DESC", + "ENCODING", + "FROM", + "FILL", + "FLOAT", + "FLUSH", + "FIRST_VALUE", + "FULL", + "FALSE", + "FOR", + "FUNCTION", + "FUNCTIONS", + "GRANT", + "GROUP", + "GORILLA", + "GLOBAL", + "GZIP", + "INSERT", + "INTO", + "INT32", + "INT64", + "INDEX", + "INFO", + "KILL", + "LIMIT", + "LINEAR", + "LABEL", + "LINK", + "LIST", + "LOAD", + "LEVEL", + "LAST_VALUE", + "LAST", + "LZO", + "LZ4", + "LATEST", + "LIKE", + "METADATA", + "MERGE", + "MOVE", + "MIN_TIME", + "MAX_TIME", + "MIN_VALUE", + "MAX_VALUE", + "NOW", + "NODES", + "ORDER", + "OFFSET", + "ON", + "OFF", + "OF", + "FLEA", + "PROCESSLIST", + "PREVIOUS", + "PREVIOUSUNTILLAST", + "PROPERTY", + "PLAIN", + "PLAIN_DICTIONARY", + "PRIVILEGES", + "PASSWORD", + "PATHS", + "PAA", + "PLA", + "PARTITION", + "QUERY", + "ROOT", + "RLE", + "REGULAR", + "ROLE", + "REVOKE", + "REMOVE", + "RENAME", + "SELECT", + "SHOW", + "SET", + "SLIMIT", + "SOFFSET", + "STORAGE", + "SUM", + "SNAPPY", + "SNAPSHOT", + "SCHEMA", + "TO", + "TIMESERIES", + "TIMESTAMP", + "TEXT", + "TS_2DIFF", + "TRACING", + "TTL", + "TASK", + "TIME", + "TAGS", + "TRUE", + "TEMPORARY", + "TOP", + "TOLERANCE", + "UPDATE", + "UNLINK", + "UPSERT", + "USING", + "USER", + "UNSET", + "UNCOMPRESSED", + "VALUES", + "VERSION", + "WHERE", + "WITH", + "WATERMARK_EMBEDDING" }; String[] sql92Keywords = { - "ABSOLUTE", - "EXEC", - "OVERLAPS", - "ACTION", - "EXECUTE", - "PAD", - "ADA", - "EXISTS", - "PARTIAL", - "ADD", - "EXTERNAL", - "PASCAL", - "ALL", - "EXTRACT", - "POSITION", - "ALLOCATE", - "FALSE", - "PRECISION", - "ALTER", - "FETCH", - "PREPARE", - "AND", - "FIRST", - "PRESERVE", - "ANY", - "FLOAT", - "PRIMARY", - "ARE", - "FOR", - "PRIOR", - "AS", - "FOREIGN", - "PRIVILEGES", - "ASC", - "FORTRAN", - "PROCEDURE", - "ASSERTION", - "FOUND", - "PUBLIC", - "AT", - "FROM", - "READ", - "AUTHORIZATION", - "FULL", - "REAL", - "AVG", - "GET", - "REFERENCES", - "BEGIN", - "GLOBAL", - "RELATIVE", - "BETWEEN", - "GO", - "RESTRICT", - "BIT", - "GOTO", - "REVOKE", - "BIT_LENGTH", - "GRANT", - "RIGHT", - "BOTH", - "GROUP", - "ROLLBACK", - "BY", - "HAVING", - "ROWS", - "CASCADE", - "HOUR", - "SCHEMA", - "CASCADED", - "IDENTITY", - "SCROLL", - "CASE", - "IMMEDIATE", - "SECOND", - "CAST", - "IN", - "SECTION", - "CATALOG", - "INCLUDE", - "SELECT", - "CHAR", - "INDEX", - "SESSION", - "CHAR_LENGTH", - "INDICATOR", - "SESSION_USER", - "CHARACTER", - "INITIALLY", - "SET", - "CHARACTER_LENGTH", - "INNER", - "SIZE", - "CHECK", - "INPUT", - "SMALLINT", - "CLOSE", - "INSENSITIVE", - "SOME", - "COALESCE", - "INSERT", - "SPACE", - "COLLATE", - "INT", - "SQL", - "COLLATION", - "INTEGER", - "SQLCA", - "COLUMN", - "INTERSECT", - "SQLCODE", - "COMMIT", - "INTERVAL", - "SQLERROR", - "CONNECT", - "INTO", - "SQLSTATE", - "CONNECTION", - "IS", - "SQLWARNING", - "CONSTRAINT", - "ISOLATION", - "SUBSTRING", - "CONSTRAINTS", - "JOIN", - "SUM", - "CONTINUE", - "KEY", - "SYSTEM_USER", - "CONVERT", - "LANGUAGE", - "TABLE", - "CORRESPONDING", - "LAST", - "TEMPORARY", - "COUNT", - "LEADING", - "THEN", - "CREATE", - "LEFT", - "TIME", - "CROSS", - "LEVEL", - "TIMESTAMP", - "CURRENT", - "LIKE", - "TIMEZONE_HOUR", - "CURRENT_DATE", - "LOCAL", - "TIMEZONE_MINUTE", - "CURRENT_TIME", - "LOWER", - "TO", - "CURRENT_TIMESTAMP", - "MATCH", - "TRAILING", - "CURRENT_USER", - "MAX", - "TRANSACTION", - "CURSOR", - "MIN", - "TRANSLATE", - "DATE", - "MINUTE", - "TRANSLATION", - "DAY", - "MODULE", - "TRIM", - "DEALLOCATE", - "MONTH", - "TRUE", - "DEC", - "NAMES", - "UNION", - "DECIMAL", - "NATIONAL", - "UNIQUE", - "DECLARE", - "NATURAL", - "UNKNOWN", - "DEFAULT", - "NCHAR", - "UPDATE", - "DEFERRABLE", - "NEXT", - "UPPER", - "DEFERRED", - "NO", - "USAGE", - "DELETE", - "NONE", - "USER", - "DESC", - "NOT", - "USING", - "DESCRIBE", - "NULL", - "VALUE", - "DESCRIPTOR", - "NULLIF", - "VALUES", - "DIAGNOSTICS", - "NUMERIC", - "VARCHAR", - "DISCONNECT", - "OCTET_LENGTH", - "VARYING", - "DISTINCT", - "OF", - "VIEW", - "DOMAIN", - "ON", - "WHEN", - "DOUBLE", - "ONLY", - "WHENEVER", - "DROP", - "OPEN", - "WHERE", - "ELSE", - "OPTION", - "WITH", - "END", - "OR", - "WORK", - "END-EXEC", - "ORDER", - "WRITE", - "ESCAPE", - "OUTER", - "YEAR", - "EXCEPT", - "OUTPUT", - "ZONE", - "EXCEPTION" + "ABSOLUTE", + "EXEC", + "OVERLAPS", + "ACTION", + "EXECUTE", + "PAD", + "ADA", + "EXISTS", + "PARTIAL", + "ADD", + "EXTERNAL", + "PASCAL", + "ALL", + "EXTRACT", + "POSITION", + "ALLOCATE", + "FALSE", + "PRECISION", + "ALTER", + "FETCH", + "PREPARE", + "AND", + "FIRST", + "PRESERVE", + "ANY", + "FLOAT", + "PRIMARY", + "ARE", + "FOR", + "PRIOR", + "AS", + "FOREIGN", + "PRIVILEGES", + "ASC", + "FORTRAN", + "PROCEDURE", + "ASSERTION", + "FOUND", + "PUBLIC", + "AT", + "FROM", + "READ", + "AUTHORIZATION", + "FULL", + "REAL", + "AVG", + "GET", + "REFERENCES", + "BEGIN", + "GLOBAL", + "RELATIVE", + "BETWEEN", + "GO", + "RESTRICT", + "BIT", + "GOTO", + "REVOKE", + "BIT_LENGTH", + "GRANT", + "RIGHT", + "BOTH", + "GROUP", + "ROLLBACK", + "BY", + "HAVING", + "ROWS", + "CASCADE", + "HOUR", + "SCHEMA", + "CASCADED", + "IDENTITY", + "SCROLL", + "CASE", + "IMMEDIATE", + "SECOND", + "CAST", + "IN", + "SECTION", + "CATALOG", + "INCLUDE", + "SELECT", + "CHAR", + "INDEX", + "SESSION", + "CHAR_LENGTH", + "INDICATOR", + "SESSION_USER", + "CHARACTER", + "INITIALLY", + "SET", + "CHARACTER_LENGTH", + "INNER", + "SIZE", + "CHECK", + "INPUT", + "SMALLINT", + "CLOSE", + "INSENSITIVE", + "SOME", + "COALESCE", + "INSERT", + "SPACE", + "COLLATE", + "INT", + "SQL", + "COLLATION", + "INTEGER", + "SQLCA", + "COLUMN", + "INTERSECT", + "SQLCODE", + "COMMIT", + "INTERVAL", + "SQLERROR", + "CONNECT", + "INTO", + "SQLSTATE", + "CONNECTION", + "IS", + "SQLWARNING", + "CONSTRAINT", + "ISOLATION", + "SUBSTRING", + "CONSTRAINTS", + "JOIN", + "SUM", + "CONTINUE", + "KEY", + "SYSTEM_USER", + "CONVERT", + "LANGUAGE", + "TABLE", + "CORRESPONDING", + "LAST", + "TEMPORARY", + "COUNT", + "LEADING", + "THEN", + "CREATE", + "LEFT", + "TIME", + "CROSS", + "LEVEL", + "TIMESTAMP", + "CURRENT", + "LIKE", + "TIMEZONE_HOUR", + "CURRENT_DATE", + "LOCAL", + "TIMEZONE_MINUTE", + "CURRENT_TIME", + "LOWER", + "TO", + "CURRENT_TIMESTAMP", + "MATCH", + "TRAILING", + "CURRENT_USER", + "MAX", + "TRANSACTION", + "CURSOR", + "MIN", + "TRANSLATE", + "DATE", + "MINUTE", + "TRANSLATION", + "DAY", + "MODULE", + "TRIM", + "DEALLOCATE", + "MONTH", + "TRUE", + "DEC", + "NAMES", + "UNION", + "DECIMAL", + "NATIONAL", + "UNIQUE", + "DECLARE", + "NATURAL", + "UNKNOWN", + "DEFAULT", + "NCHAR", + "UPDATE", + "DEFERRABLE", + "NEXT", + "UPPER", + "DEFERRED", + "NO", + "USAGE", + "DELETE", + "NONE", + "USER", + "DESC", + "NOT", + "USING", + "DESCRIBE", + "NULL", + "VALUE", + "DESCRIPTOR", + "NULLIF", + "VALUES", + "DIAGNOSTICS", + "NUMERIC", + "VARCHAR", + "DISCONNECT", + "OCTET_LENGTH", + "VARYING", + "DISTINCT", + "OF", + "VIEW", + "DOMAIN", + "ON", + "WHEN", + "DOUBLE", + "ONLY", + "WHENEVER", + "DROP", + "OPEN", + "WHERE", + "ELSE", + "OPTION", + "WITH", + "END", + "OR", + "WORK", + "END-EXEC", + "ORDER", + "WRITE", + "ESCAPE", + "OUTER", + "YEAR", + "EXCEPT", + "OUTPUT", + "ZONE", + "EXCEPTION" }; TreeMap myKeywordMap = new TreeMap(); for (int i = 0; i < allIotdbSQLKeywords.length; i++) { @@ -487,12 +487,11 @@ private WatermarkEncoder getWatermarkEncoder() { try { - groupedLSBWatermarkEncoder = - new GroupedLSBWatermarkEncoder( - client.getProperties().getWatermarkSecretKey(), - client.getProperties().getWatermarkBitString(), - client.getProperties().getWatermarkParamMarkRate(), - client.getProperties().getWatermarkParamMaxRightBit()); + groupedLSBWatermarkEncoder = new GroupedLSBWatermarkEncoder( + client.getProperties().getWatermarkSecretKey(), + client.getProperties().getWatermarkBitString(), + client.getProperties().getWatermarkParamMarkRate(), + client.getProperties().getWatermarkParamMaxRightBit()); } catch (TException e) { e.printStackTrace(); } @@ -878,8 +877,7 @@ for (Map<String, Object> map : listPaths) { TSDataType columnType = (TSDataType) map.get("type"); Object val = map.get("val"); - org.apache.iotdb.tsfile.read.common.Field field = - new org.apache.iotdb.tsfile.read.common.Field(columnType); + org.apache.iotdb.tsfile.read.common.Field field = new org.apache.iotdb.tsfile.read.common.Field(columnType); switch (columnType) { case TEXT: field.setBinaryV(new Binary(val.toString())); @@ -922,8 +920,8 @@ fields[1] = new Field("", "MAX_LEN", "INT32"); fields[2] = new Field("", "DEFAULT_VALUE", "INT32"); fields[3] = new Field("", "DESCRIPTION", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList(TSDataType.TEXT, TSDataType.INT32, TSDataType.INT32, TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList(TSDataType.TEXT, TSDataType.INT32, TSDataType.INT32, + TSDataType.TEXT); List<Object> values = Arrays.asList("fetch_size", 10, 10, ""); List<String> columnNameList = new ArrayList<>(); @@ -981,7 +979,7 @@ sql = sql + " " + schemaPattern; } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0) { if (tableNamePattern.contains("%")) { @@ -991,7 +989,7 @@ } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0 && columnNamePattern != null @@ -1008,16 +1006,15 @@ fields[5] = new Field("", "GRANTEE", "TEXT"); fields[6] = new Field("", "PRIVILEGE", "TEXT"); fields[7] = new Field("", "IS_GRANTABLE", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); @@ -1272,25 +1269,24 @@ fields[14] = new Field("", "ORDINAL_POSITION", "INT32"); fields[15] = new Field("", "IS_NULLABLE", "TEXT"); fields[16] = new Field("", "SPECIFIC_NAME", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); @@ -1352,14 +1348,13 @@ fields[3] = new Field("", "REMARKS", "TEXT"); fields[4] = new Field("", "FUNCTION_TYPE", "INT32"); fields[5] = new Field("", "SPECIFIC_NAME", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); @@ -1673,14 +1668,13 @@ List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); Map<String, Integer> columnNameIndex = new HashMap<>(); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); Field[] fields = new Field[6]; fields[0] = new Field("", "TABLE_CAT", "TEXT"); fields[1] = new Field("", "TABLE_SCHEM", "TEXT"); @@ -1841,24 +1835,22 @@ fields[9] = new Field("", "REMARKS", "TEXT"); fields[10] = new Field("", "CHAR_OCTET_LENGTH", "INT32"); fields[11] = new Field("", "IS_NULLABLE", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT); - List<Object> value = - Arrays.asList( - catalog, catalog, tableNamePattern, "times", Types.BIGINT, 1, 0, 2, "", "", 13, "NO"); + List<Object> value = Arrays.asList( + catalog, catalog, tableNamePattern, "times", Types.BIGINT, 1, 0, 2, "", "", 13, "NO"); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); Map<String, Integer> columnNameIndex = new HashMap<>(); @@ -2106,7 +2098,7 @@ sql = sql + " " + schemaPattern; } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0) { if (tableNamePattern.contains("%")) { @@ -2125,16 +2117,15 @@ fields[5] = new Field("", "GRANTEE", "TEXT"); fields[6] = new Field("", "PRIVILEGE", "TEXT"); fields[7] = new Field("", "IS_GRANTABLE", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); @@ -2247,7 +2238,7 @@ sql = sql + " " + schemaPattern; } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0) { if (tableNamePattern.contains("%")) { @@ -2257,7 +2248,7 @@ } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0 && columnNamePattern != null @@ -2291,32 +2282,31 @@ fields[22] = new Field("", "IS_AUTOINCREMENT", "TEXT"); fields[23] = new Field("", "IS_GENERATEDCOLUMN", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); Map<String, Integer> columnNameIndex = new HashMap<>(); @@ -2495,7 +2485,7 @@ sql = sql + " " + schemaPattern; } if (((catalog != null && catalog.length() > 0) - || schemaPattern != null && schemaPattern.length() > 0) + || schemaPattern != null && schemaPattern.length() > 0) && tableNamePattern != null && tableNamePattern.length() > 0) { if (tableNamePattern.contains("%")) { @@ -2516,18 +2506,17 @@ fields[8] = new Field("", "SELF_REFERENCING_COL_NAME", "TEXT"); fields[9] = new Field("", "REF_GENERATION", "TEXT"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); @@ -2604,149 +2593,141 @@ fields[15] = new Field("", "SQL_DATA_TYPE", "INT32"); fields[16] = new Field("", "SQL_DATETIME_SUB", "INT32"); fields[17] = new Field("", "NUM_PREC_RADIX", "INT32"); - List<TSDataType> tsDataTypeList = - Arrays.asList( - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.BOOLEAN, - TSDataType.TEXT, - TSDataType.BOOLEAN, - TSDataType.BOOLEAN, - TSDataType.BOOLEAN, - TSDataType.TEXT, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32, - TSDataType.INT32); - List<Object> listValSub_1 = - Arrays.asList( - "INT32", - Types.INTEGER, - 10, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<Object> listValSub_2 = - Arrays.asList( - "INT64", - Types.BIGINT, - 19, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<Object> listValSub_3 = - Arrays.asList( - "BOOLEAN", - Types.BOOLEAN, - 1, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<Object> listValSub_4 = - Arrays.asList( - "FLOAT", - Types.FLOAT, - 38, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<Object> listValSub_5 = - Arrays.asList( - "DOUBLE", - Types.DOUBLE, - 308, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<Object> listValSub_6 = - Arrays.asList( - "TEXT", - Types.LONGVARCHAR, - 64, - "", - "", - "", - 1, - true, - "", - false, - true, - false, - "", - 0, - 10, - 0, - 0, - 10); - List<List<Object>> valuesList = - Arrays.asList( - listValSub_1, listValSub_2, listValSub_3, listValSub_4, listValSub_5, listValSub_6); + List<TSDataType> tsDataTypeList = Arrays.asList( + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.TEXT, + TSDataType.BOOLEAN, + TSDataType.BOOLEAN, + TSDataType.BOOLEAN, + TSDataType.TEXT, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32, + TSDataType.INT32); + List<Object> listValSub_1 = Arrays.asList( + "INT32", + Types.INTEGER, + 10, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<Object> listValSub_2 = Arrays.asList( + "INT64", + Types.BIGINT, + 19, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<Object> listValSub_3 = Arrays.asList( + "BOOLEAN", + Types.BOOLEAN, + 1, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<Object> listValSub_4 = Arrays.asList( + "FLOAT", + Types.FLOAT, + 38, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<Object> listValSub_5 = Arrays.asList( + "DOUBLE", + Types.DOUBLE, + 308, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<Object> listValSub_6 = Arrays.asList( + "TEXT", + Types.LONGVARCHAR, + 64, + "", + "", + "", + 1, + true, + "", + false, + true, + false, + "", + 0, + 10, + 0, + 0, + 10); + List<List<Object>> valuesList = Arrays.asList( + listValSub_1, listValSub_2, listValSub_3, listValSub_4, listValSub_5, listValSub_6); List<String> columnNameList = new ArrayList<>(); List<String> columnTypeList = new ArrayList<>(); Map<String, Integer> columnNameIndex = new HashMap<>();
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 7b303e3..a9c25f5 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
@@ -50,10 +50,10 @@ public class SchemaUtils { - private SchemaUtils() {} + private SchemaUtils() { + } - private static final Map<TSDataType, Set<TSEncoding>> schemaChecker = - new EnumMap<>(TSDataType.class); + private static final Map<TSDataType, Set<TSEncoding>> schemaChecker = new EnumMap<>(TSDataType.class); static { Set<TSEncoding> booleanSet = new HashSet<>(); @@ -68,7 +68,7 @@ intSet.add(TSEncoding.GORILLA); intSet.add(TSEncoding.ZIGZAG); intSet.add(TSEncoding.FREQ); - intSet.add(TSEncoding.PERIOD); + intSet.add(TSEncoding.FLEA); schemaChecker.put(TSDataType.INT32, intSet); schemaChecker.put(TSDataType.INT64, intSet); @@ -118,8 +118,10 @@ } /** - * If the datatype of 'aggregation' depends on 'measurementDataType' (min_value, max_value), - * return 'measurementDataType' directly, or return a list whose elements are all the datatype of + * If the datatype of 'aggregation' depends on 'measurementDataType' (min_value, + * max_value), + * return 'measurementDataType' directly, or return a list whose elements are + * all the datatype of * 'aggregation' and its length is the same as 'measurementDataType'. * * @param measurementDataType @@ -186,7 +188,8 @@ } /** - * judge whether the order of aggregation calculation is consistent with the order of traversing + * judge whether the order of aggregation calculation is consistent with the + * order of traversing * data */ public static boolean isConsistentWithScanOrder( @@ -213,7 +216,8 @@ } /** - * If e or one of its recursive causes is a PathNotExistException or StorageGroupNotSetException, + * If e or one of its recursive causes is a PathNotExistException or + * StorageGroupNotSetException, * return such an exception or null if it cannot be found. * * @param currEx
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java index a459f98..69d4f12 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/Decoder.java
@@ -123,11 +123,11 @@ } case FREQ: return new FreqDecoder(); - case PERIOD: + case FLEA: switch (dataType) { case INT32: // Spade TODO: replace plain with period parameter - return new PeriodDecoder(); + return new FLEADecoder(); default: throw new TsFileDecodingException(String.format(ERROR_MSG, encoding, dataType)); }
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FLEADecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FLEADecoder.java new file mode 100644 index 0000000..d98132b --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/FLEADecoder.java
@@ -0,0 +1,434 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.encoding.decoder; + +import org.apache.iotdb.tsfile.exception.encoding.TsFileDecodingException; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Binary; + +import org.apache.commons.math3.complex.Complex; +import org.apache.commons.math3.transform.DftNormalization; +import org.apache.commons.math3.transform.FastFourierTransformer; +import org.apache.commons.math3.transform.TransformType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; + +public class FLEADecoder extends Decoder { + + private static final Logger logger = LoggerFactory.getLogger(FLEADecoder.class); + private Queue<Integer> decodedValues; + + public FLEADecoder() { + super(TSEncoding.FLEA); + this.decodedValues = new ArrayDeque<>(); + } + + @Override + public int readInt(ByteBuffer buffer) { + if (!hasNext(buffer)) { + throw new TsFileDecodingException("No more values to read."); + } + return decodedValues.poll(); + } + + @Override + public boolean hasNext(ByteBuffer buffer) { + if (!decodedValues.isEmpty()) { + return true; + } + if (buffer.remaining() > 0) { + decodeNextBlock(buffer); + return !decodedValues.isEmpty(); + } + return false; + } + + private void decodeNextBlock(ByteBuffer buffer) { + if (buffer.remaining() < 4) { + return; + } + int blockLength = buffer.getInt(); + + if (buffer.remaining() < blockLength) { + throw new TsFileDecodingException( + "Error when decoding FLEA block: expected " + + blockLength + + " bytes, but only " + + buffer.remaining() + + " are available."); + } + + byte[] blockBytes = new byte[blockLength]; + buffer.get(blockBytes); + + try { + int[] blockOfInts = decodeBlock(blockBytes); + for (int val : blockOfInts) { + decodedValues.add(val); + } + } catch (IOException e) { + throw new TsFileDecodingException("Error when decoding FLEA block.", e); + } + } + + private int[] decodeBlock(byte[] blockBytes) throws IOException { + ByteArrayInputStream bais = new ByteArrayInputStream(blockBytes); + + OptimalParams params = decodeMetadata(bais); + int n = params.n; + + Complex[] F_hat_complex_dequant = decodeFrequencyComponent(params, bais); + + int[] R_star = decodeResidualComponent(params, bais); + + Complex[] F_hat_full = new Complex[n]; + for (int j = 0; j < F_hat_complex_dequant.length; j++) { + F_hat_full[j] = F_hat_complex_dequant[j]; + } + for (int j = 1; j < F_hat_complex_dequant.length; j++) { + if (n - j < n) { + F_hat_full[n - j] = F_hat_complex_dequant[j].conjugate(); + } + } + + FastFourierTransformer fft = new FastFourierTransformer(DftNormalization.STANDARD); + Complex[] reconstructedTimeComplex = fft.transform(F_hat_full, TransformType.INVERSE); + + int[] finalValues = new int[n]; + for (int k = 0; k < n; k++) { + finalValues[k] = (int) Math.round(reconstructedTimeComplex[k].getReal() + R_star[k]); + } + + return finalValues; + } + + private static class OptimalParams { + int n; + int betaStar; + int pReStar; + int pImStar; + int dResidualStar; + } + + private OptimalParams decodeMetadata(ByteArrayInputStream bais) throws IOException { + + final int METADATA_SIZE_BYTES = 20; + byte[] metaBytes = new byte[METADATA_SIZE_BYTES]; + + int bytesRead = bais.read(metaBytes); + if (bytesRead != METADATA_SIZE_BYTES) { + throw new IOException( + "Failed to read complete metadata block. Expected " + + METADATA_SIZE_BYTES + + " bytes, but got " + + bytesRead); + } + + ByteBuffer metaBuffer = ByteBuffer.wrap(metaBytes); + + OptimalParams params = new OptimalParams(); + params.n = metaBuffer.getInt(); + params.betaStar = metaBuffer.getInt(); + params.pReStar = metaBuffer.getInt(); + params.pImStar = metaBuffer.getInt(); + params.dResidualStar = metaBuffer.getInt(); + + logger.debug( + "Decoded metadata: n={}, beta={}, p_re={}, p_im={}, d_res={}", + params.n, + params.betaStar, + params.pReStar, + params.pImStar, + params.dResidualStar); + + if (params.n <= 0 + || params.betaStar < 0 + || params.pReStar < 0 + || params.pImStar < 0 + || params.dResidualStar < 0) { + throw new TsFileDecodingException("Invalid metadata decoded: " + params); + } + + return params; + } + + private Complex[] decodeFrequencyComponent(OptimalParams params, ByteArrayInputStream bais) + throws IOException { + BitInputStream bitIn = new BitInputStream(bais); + + int n_half = params.n / 2 + 1; + + int[] F_re_desc = decodeGVLE(params.pReStar, bitIn); + int[] F_re_sparse = decodeDBP(n_half - params.pReStar, bitIn); + + int[] F_im_desc = decodeGVLE(params.pImStar, bitIn); + int[] F_im_sparse = decodeDBP(n_half - params.pImStar, bitIn); + + bitIn.close(); + + int[] F_hat_re = new int[n_half]; + System.arraycopy(F_re_desc, 0, F_hat_re, 0, F_re_desc.length); + if (F_re_sparse.length > 0) + System.arraycopy(F_re_sparse, 0, F_hat_re, F_re_desc.length, F_re_sparse.length); + int[] F_hat_im = new int[n_half]; + System.arraycopy(F_im_desc, 0, F_hat_im, 0, F_im_desc.length); + if (F_im_sparse.length > 0) + System.arraycopy(F_im_sparse, 0, F_hat_im, F_im_desc.length, F_im_sparse.length); + + double quantizationFactor = Math.pow(2, params.betaStar); + Complex[] dequantizedFreq = new Complex[n_half]; + for (int j = 0; j < n_half; j++) { + dequantizedFreq[j] = + new Complex( + (double) F_hat_re[j] * quantizationFactor, (double) F_hat_im[j] * quantizationFactor); + } + return dequantizedFreq; + } + + private int[] decodeGVLE(int totalValuesToRead, BitInputStream bitIn) throws IOException { + if (totalValuesToRead == 0) { + return new int[0]; + } + + List<Integer> decodedList = new ArrayList<>(totalValuesToRead); + int valuesRead = 0; + + while (valuesRead < totalValuesToRead) { + int sharedWidth = bitIn.read(5); + int groupLength = bitIn.read(11); + + if (valuesRead + groupLength > totalValuesToRead) { + throw new TsFileDecodingException( + "GVLE decoding error: group length " + + groupLength + + " exceeds remaining values to read " + + (totalValuesToRead - valuesRead)); + } + + for (int i = 0; i < groupLength; i++) { + int sign = bitIn.read(1); + + int magnitude = 0; + if (sharedWidth > 0) { + magnitude = bitIn.read(sharedWidth); + } + + int value = (sign == 1) ? -magnitude : magnitude; + decodedList.add(value); + } + + valuesRead += groupLength; + } + + return decodedList.stream().mapToInt(i -> i).toArray(); + } + + private static class DBPValue { + int value; + int position; + + DBPValue(int value, int position) { + this.value = value; + this.position = position; + } + } + + private int[] decodeDBP(int segmentLength, BitInputStream bitIn) throws IOException { + int nonZeroCount = bitIn.read(16); + + if (nonZeroCount == 0) { + return new int[segmentLength]; + } + + List<Integer> positions = new ArrayList<>(nonZeroCount); + int posWidth = (segmentLength == 0) ? 0 : 32 - Integer.numberOfLeadingZeros(segmentLength - 1); + if (posWidth == 0) posWidth = 1; + + for (int i = 0; i < nonZeroCount; i++) { + positions.add(bitIn.read(posWidth)); + } + + List<Integer> values = new ArrayList<>(nonZeroCount); + + int prevValWidth = bitIn.read(5); + + int sign = bitIn.read(1); + int magnitude = bitIn.read(prevValWidth); + int firstVal = (sign == 1) ? -magnitude : magnitude; + values.add(firstVal); + + for (int i = 1; i < nonZeroCount; i++) { + int currentSign = bitIn.read(1); + int currentMagnitude = bitIn.read(prevValWidth); + int currentVal = (currentSign == 1) ? -currentMagnitude : currentMagnitude; + values.add(currentVal); + + int currentAbsVal = Math.abs(currentVal); + prevValWidth = (currentAbsVal == 0) ? 1 : 32 - Integer.numberOfLeadingZeros(currentAbsVal); + } + + int[] sequence = new int[segmentLength]; + for (int i = 0; i < nonZeroCount; i++) { + int pos = positions.get(i); + int val = values.get(i); + if (pos < segmentLength) { + sequence[pos] = val; + } else { + throw new TsFileDecodingException( + "Decoded position " + pos + " is out of bounds for segment length " + segmentLength); + } + } + + return sequence; + } + + private int[] decodeResidualComponent(OptimalParams params, ByteArrayInputStream bais) + throws IOException { + BitInputStream bitIn = new BitInputStream(bais); + int[] residuals = decodeHybridResiduals(params.n, params.dResidualStar, bitIn); + return residuals; + } + + private int[] decodeHybridResiduals(int n, int optimalD, BitInputStream bitIn) + throws IOException { + if (n == 0) { + return new int[0]; + } + + int[] residuals = new int[n]; + int lowBitMask = (1 << optimalD) - 1; + + for (int i = 0; i < n; i++) { + int sign = bitIn.read(1); + + int lowMagnitude = 0; + if (optimalD > 0) { + lowMagnitude = bitIn.read(optimalD); + } + + residuals[i] = (sign == 1) ? -lowMagnitude : lowMagnitude; + } + + int[] highBitSequence = decodeDBP(n, bitIn); + + for (int i = 0; i < n; i++) { + if (highBitSequence[i] != 0) { + int signedHighPart = highBitSequence[i]; + int absHighPart = Math.abs(signedHighPart); + + int absLowPart = Math.abs(residuals[i]); + int fullMagnitude = (absHighPart << optimalD) | absLowPart; + + residuals[i] = (signedHighPart < 0) ? -fullMagnitude : fullMagnitude; + } + } + + return residuals; + } + + private static class BitInputStream { + + private final ByteArrayInputStream byteInput; + private int bitBuffer; + private int bitsAvailable; + + public BitInputStream(ByteArrayInputStream byteInput) { + this.byteInput = byteInput; + this.bitBuffer = 0; + this.bitsAvailable = 0; + } + + public int read(int numBits) throws IOException { + if (numBits <= 0 || numBits > 32) { + throw new IllegalArgumentException("Number of bits must be between 1 and 32."); + } + + while (bitsAvailable < numBits) { + int nextByte = byteInput.read(); + if (nextByte == -1) { + throw new IOException( + "Unexpected end of stream while trying to read " + numBits + " bits."); + } + bitBuffer = (bitBuffer << 8) | (nextByte & 0xFF); + bitsAvailable += 8; + } + + int value = (bitBuffer >> (bitsAvailable - numBits)); + + bitsAvailable -= numBits; + + int mask = (1 << bitsAvailable) - 1; + bitBuffer &= mask; + + return value; + } + + public void close() {} + } + + @Override + public boolean readBoolean(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public short readShort(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public long readLong(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public float readFloat(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public double readDouble(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public Binary readBinary(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public BigDecimal readBigDecimal(ByteBuffer buffer) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void reset() { + decodedValues.clear(); + } +}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PeriodDecoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PeriodDecoder.java deleted file mode 100644 index e45a2b7..0000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/PeriodDecoder.java +++ /dev/null
@@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tsfile.encoding.decoder; - -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; - -import org.apache.commons.math3.complex.Complex; -import org.apache.commons.math3.transform.DftNormalization; -import org.apache.commons.math3.transform.FastFourierTransformer; -import org.apache.commons.math3.transform.TransformType; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class PeriodDecoder extends Decoder { - - private static final int MAX_SIZE = 0x7FFFFFFF; - private static final int MAX_VALUE = 0x7FFFFFFF; - private static final int GROUP_SIZE = 8; - - private static final FastFourierTransformer transformer = - new FastFourierTransformer(DftNormalization.STANDARD); - - private static double[] irfft(Complex[] input, int n) { - Complex[] conjugates = new Complex[n]; - - // 构造逆FFT所需的复数数组 - for (int i = 0; i < input.length; i++) { - if (i == 0 || (i == input.length - 1 && n % 2 == 0)) { - conjugates[i] = input[i]; - } else { - conjugates[i] = input[i]; - conjugates[n - i] = input[i].conjugate(); - } - } - - // 进行逆FFT，并除以长度进行归一化 - Complex[] inverse = transformer.transform(conjugates, TransformType.INVERSE); - double[] result = new double[n]; - for (int i = 0; i < n; i++) { - result[i] = inverse[i].getReal() / n; - } - - return result; - } - - private static Complex[] compRoundInverse(int[] ret, int beta) { - Complex[] result = new Complex[ret.length / 2]; - - for (int i = 0; i < ret.length; i += 2) { - double real = ret[i] * Math.pow(2, beta); - double imag = ret[i + 1] * Math.pow(2, beta); - result[i / 2] = new Complex(real, imag); - } - - return result; - } - - private static int bitLength(int value) { - return Integer.SIZE - Integer.numberOfLeadingZeros(value); - } - - private int decode(ByteBuffer buffer, int bits) { - int value = 0; - for (int i = 0; i < bits; i++) { - value = (value << 1) | (buffer.get() & 1); - } - return value; - } - - private List<Integer> descendingBitPackingDecode(ByteBuffer buffer) { - boolean sgn = decode(buffer, 1) == 1; - int dataLength = decode(buffer, bitLength(MAX_SIZE)); - List<Integer> data = new ArrayList<>(dataLength); - for (int i = 0; i < dataLength; i++) { - data.add(0); - } - int indexLength = decode(buffer, bitLength(MAX_SIZE)); - List<Integer> index = new ArrayList<>(indexLength); - for (int i = 0; i < indexLength; i++) { - index.add(0); - } - if (indexLength == 0) { - return data; - } - for (int i = 0; i < indexLength; i += GROUP_SIZE) { - int maxLen = decode(buffer, bitLength(bitLength(MAX_SIZE))); - for (int j = i; j < Math.min(i + GROUP_SIZE, indexLength); j++) { - index.set(j, decode(buffer, maxLen)); - } - } - int firstLen = decode(buffer, bitLength(bitLength(MAX_VALUE))); - int currentLen = firstLen; - for (int i = 0; i < indexLength; i++) { - boolean currentSgn = sgn && decode(buffer, 1) == 1; - int value = decode(buffer, currentLen); - data.set(index.get(i), currentSgn ? -value : value); - currentLen = bitLength(value); - } - return data; - } - - private List<Integer> separateStorageDecode(ByteBuffer buffer) { - int dataLength = decode(buffer, bitLength(MAX_SIZE)); - List<Integer> sgn = new ArrayList<>(dataLength); - List<Integer> data = new ArrayList<>(dataLength); - for (int i = 0; i < dataLength; i++) { - sgn.add(0); - data.add(0); - } - int D = decode(buffer, bitLength(bitLength(MAX_VALUE))); - for (int i = 0; i < dataLength; i++) { - sgn.set(i, decode(buffer, 1)); - data.set(i, decode(buffer, D)); - } - List<Integer> high = descendingBitPackingDecode(buffer); - for (int i = 0; i < dataLength; i++) { - data.set(i, (sgn.get(i) == 0 ? 1 : -1) * ((high.get(i) << D) | data.get(i))); - } - return data; - } - - public List<Integer> periodDecode(ByteBuffer buffer) throws IOException { - int p = decode(buffer, bitLength(MAX_SIZE)); - if (p == 0) { - return separateStorageDecode(buffer); - } - boolean betaSgn = decode(buffer, 1) == 1; - int beta = decode(buffer, bitLength(bitLength(MAX_VALUE))); - if (betaSgn) { - beta *= -1; - } - Complex[] dataf = - compRoundInverse( - descendingBitPackingDecode(buffer).stream().mapToInt(i -> i).toArray(), beta); - List<Integer> res = cumulativeSum(separateStorageDecode(buffer)); - int k = (res.size() + p - 1) / p; - double[] tmp = irfft(dataf, p); - List<Integer> tiledList = new ArrayList<>(); - for (int j = 0; j < k; j++) for (int i = 0; i < tmp.length; i++) tiledList.add((int) tmp[i]); - return IntStream.range(0, res.size()) - .map(i -> tiledList.get(i) - res.get(i)) - .boxed() - .collect(Collectors.toList()); - } - - // 辅助方法: 累积求和 - private List<Integer> cumulativeSum(List<Integer> list) { - int sum = 0; - for (int i = 0; i < list.size(); i++) { - sum += list.get(i); - list.set(i, sum); - } - return list; - } - - List<Integer> data; - int curPos; - boolean isRead; - - public PeriodDecoder() { - super(TSEncoding.PERIOD); - reset(); - } - - // @Override - // public boolean readBoolean(ByteBuffer buffer) { - // return buffer.get() != 0; - // } - - // @Override - // public short readShort(ByteBuffer buffer) { - // return buffer.getShort(); - // } - - @Override - public int readInt(ByteBuffer buffer) { - try { - if (isRead == false) { - data = periodDecode(buffer); - isRead = true; - curPos = 0; - } - curPos++; - return data.get(curPos - 1); - } catch (Exception e) { - return -1; - } - } - - // @Override - // public long readLong(ByteBuffer buffer) { - // return buffer.getLong(); - // } - - // @Override - // public float readFloat(ByteBuffer buffer) { - // return buffer.getFloat(); - // } - - // @Override - // public double readDouble(ByteBuffer buffer) { - // return buffer.getDouble(); - // } - - // @Override - // public Binary readBinary(ByteBuffer buffer) { - // int length = readInt(buffer); - // byte[] buf = new byte[length]; - // buffer.get(buf, 0, buf.length); - // return new Binary(buf); - // } - - @Override - public boolean hasNext(ByteBuffer buffer) throws IOException { - if (isRead == false) { - data = periodDecode(buffer); - isRead = true; - curPos = 0; - } - return curPos < data.size(); - } - - // @Override - // public BigDecimal readBigDecimal(ByteBuffer buffer) { - // throw new TsFileDecodingException("Method readBigDecimal is not supported by - // PeriodDecoder"); - // } - - @Override - public void reset() { - isRead = false; - } -}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FLEAEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FLEAEncoder.java new file mode 100644 index 0000000..95b44f9 --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/FLEAEncoder.java
@@ -0,0 +1,681 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.encoding.encoder; + +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Binary; + +import org.apache.commons.math3.complex.Complex; +import org.apache.commons.math3.transform.DftNormalization; +import org.apache.commons.math3.transform.FastFourierTransformer; +import org.apache.commons.math3.transform.TransformType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class FLEAEncoder extends Encoder { + + private static final Logger logger = LoggerFactory.getLogger(FLEAEncoder.class); + private final TSDataType dataType; + + private final List<Integer> valuesBuffer; + private final int blockSize; + + private static final int DEFAULT_BLOCK_SIZE = 256; + private static final int MAX_BETA_CANDIDATES = 10; + + public FLEAEncoder() { + this(TSDataType.INT32, DEFAULT_BLOCK_SIZE); + } + + public FLEAEncoder(TSDataType dataType, int blockSize) { + super(TSEncoding.FLEA); + if (dataType != TSDataType.INT32) { + throw new UnsupportedOperationException( + "FLEAEncoder currently only supports INT32 data type."); + } + this.dataType = dataType; + this.valuesBuffer = new ArrayList<>(); + this.blockSize = blockSize; + logger.info("FLEAEncoder initialized for INT32 with block size: {}", this.blockSize); + } + + @Override + public void encode(int value, ByteArrayOutputStream out) { + valuesBuffer.add(value); + if (valuesBuffer.size() >= blockSize) { + try { + flush(out); + } catch (IOException e) { + logger.error("FLEAEncoder: IOException during flush operation for INT32.", e); + } + } + } + + @Override + public void flush(ByteArrayOutputStream out) throws IOException { + if (valuesBuffer.isEmpty()) { + return; + } + + logger.debug("FLEAEncoder: Flushing {} INT32 values.", valuesBuffer.size()); + + double[] blockData = valuesBuffer.stream().mapToDouble(Integer::doubleValue).toArray(); + int n = blockData.length; + + OptimalParams optimalParams = findOptimalBetaAndSplitPoints(blockData); + + byte[] encodedData = performActualEncoding(blockData, optimalParams, n); + + out.write(ByteBuffer.allocate(4).putInt(encodedData.length).array()); + out.write(encodedData); + + valuesBuffer.clear(); + } + + private static class OptimalParams { + int betaStar; + int pReStar; + int pImStar; + int dResidualStar; + + OptimalParams(int beta, int pRe, int pIm) { + this.betaStar = beta; + this.pReStar = pRe; + this.pImStar = pIm; + this.dResidualStar = 0; + } + } + + private OptimalParams findOptimalBetaAndSplitPoints(double[] blockData) { + logger.debug("Finding optimal beta and split points for block of size {}", blockData.length); + + final int n = blockData.length; + if (n == 0) { + return new OptimalParams(0, 0, 0); + } + final FastFourierTransformer fft = new FastFourierTransformer(DftNormalization.STANDARD); + final Complex[] originalFreqComplex = fft.transform(blockData, TransformType.FORWARD); + + final int n_half = (n / 2) + 1; + + double minTotalCost = Double.MAX_VALUE; + OptimalParams bestParams = new OptimalParams(0, 0, 0); + + for (int betaCandidate = 8; betaCandidate < 24; betaCandidate += 2) { + + int[] quantizedFreqRe = new int[n_half]; + int[] quantizedFreqIm = new int[n_half]; + double quantizationFactor = Math.pow(2, betaCandidate); + + for (int j = 0; j < n_half; j++) { + quantizedFreqRe[j] = + (int) Math.round(originalFreqComplex[j].getReal() / quantizationFactor); + quantizedFreqIm[j] = + (int) Math.round(originalFreqComplex[j].getImaginary() / quantizationFactor); + } + + double costResidual = + estimateResidualCost( + n, originalFreqComplex, quantizedFreqRe, quantizedFreqIm, betaCandidate); + + CostAndSplits freqCostResult = estimateFrequencyCost(quantizedFreqRe, quantizedFreqIm); + + double currentTotalCost = costResidual + freqCostResult.cost; + logger.trace( + "For beta={}: FreqCost={}, ResidCost={}, TotalCost={}", + betaCandidate, + freqCostResult.cost, + costResidual, + currentTotalCost); + + if (currentTotalCost < minTotalCost) { + minTotalCost = currentTotalCost; + bestParams.betaStar = betaCandidate; + bestParams.pReStar = freqCostResult.pRe; + bestParams.pImStar = freqCostResult.pIm; + } + } + + logger.info( + "Found optimal parameters: beta={}, p_re={}, p_im={}", + bestParams.betaStar, + bestParams.pReStar, + bestParams.pImStar); + + return bestParams; + } + + private static class CostAndSplits { + double cost; + int pRe; + int pIm; + + CostAndSplits(double c, int pr, int pi) { + this.cost = c; + this.pRe = pr; + this.pIm = pi; + } + } + + private CostAndSplits estimateFrequencyCost(int[] quantizedFreqRe, int[] quantizedFreqIm) { + logger.debug("Estimating frequency cost..."); + + SplitResult reResult = findOptimalSplitForSequence(quantizedFreqRe); + SplitResult imResult = findOptimalSplitForSequence(quantizedFreqIm); + + double totalCost = reResult.minCost + imResult.minCost; + return new CostAndSplits(totalCost, reResult.pStar, imResult.pStar); + } + + private static class SplitResult { + double minCost; + int pStar; + + SplitResult(double mc, int ps) { + this.minCost = mc; + this.pStar = ps; + } + } + + private SplitResult findOptimalSplitForSequence(int[] sequence) { + logger.debug("Finding optimal split for sequence of length {}", sequence.length); + final int n = sequence.length; + if (n == 0) { + return new SplitResult(0, 0); + } + + double[] costPrefixGVLE = new double[n + 1]; + costPrefixGVLE[0] = 0; + for (int p = 1; p <= n; p++) { + costPrefixGVLE[p] = costGVLE(Arrays.copyOfRange(sequence, 0, p)); + } + + double[] costSuffixDBP = new double[n + 1]; + costSuffixDBP[n] = 0; + for (int p = n - 1; p >= 0; p--) { + costSuffixDBP[p] = costDBP(Arrays.copyOfRange(sequence, p, n)); + } + + double minTotalCost = Double.MAX_VALUE; + int bestP = 0; + for (int p = 0; p <= n; p++) { + double currentTotalCost = costPrefixGVLE[p] + costSuffixDBP[p]; + if (currentTotalCost < minTotalCost) { + minTotalCost = currentTotalCost; + bestP = p; + } + } + + logger.trace("Found best split point p*={} with cost={}", bestP, minTotalCost); + return new SplitResult(minTotalCost, bestP); + } + + private double estimateResidualCost( + int n, Complex[] originalFreq, int[] quantizedFreqRe, int[] quantizedFreqIm, int beta) { + logger.debug("Estimating residual cost for beta={}", beta); + double quantizationFactor = Math.pow(2, beta); + double eFreq = 0; + + for (int j = 0; j < quantizedFreqRe.length; j++) { + double dequantReal = (double) quantizedFreqRe[j] * quantizationFactor; + double dequantImag = (double) quantizedFreqIm[j] * quantizationFactor; + double errorReal = dequantReal - originalFreq[j].getReal(); + double errorImag = dequantImag - originalFreq[j].getImaginary(); + + double errorMagnitudeSq = errorReal * errorReal + errorImag * errorImag; + + if (j > 0 && j < (n / 2)) { + eFreq += 2 * errorMagnitudeSq; + } else { + eFreq += errorMagnitudeSq; + } + } + + if (n == 0 || eFreq < 1e-9) return 0; + + double eR = (1.0 / n) * Math.sqrt(eFreq); + if (eR < 1e-9) eR = 1e-9; + + int de = (int) Math.ceil(Math.log(eR) / Math.log(2)); + int ds = de + 1; + + return (double) n * (ds + 1); + } + + private double costGVLE(int[] sequence) { + final int m = sequence.length; + if (m == 0) { + return 0; + } + + int[] wSuffix = new int[m]; + wSuffix[m - 1] = + (sequence[m - 1] == 0) ? 0 : (32 - Integer.numberOfLeadingZeros(Math.abs(sequence[m - 1]))); + for (int i = m - 2; i >= 0; i--) { + int currentWidth = + (sequence[i] == 0) ? 0 : (32 - Integer.numberOfLeadingZeros(Math.abs(sequence[i]))); + wSuffix[i] = Math.max(currentWidth, wSuffix[i + 1]); + } + + double totalCost = 0; + final int HEADER_COST = 16; + + int currentIdx = 0; + while (currentIdx < m) { + int sharedWidth = wSuffix[currentIdx]; + int groupStartIdx = currentIdx; + + while (currentIdx < m && wSuffix[currentIdx] == sharedWidth) { + currentIdx++; + } + int groupLength = currentIdx - groupStartIdx; + + totalCost += HEADER_COST; + totalCost += (double) groupLength * (sharedWidth + 1); + } + + return totalCost; + } + + private double costDBP(int[] sequence) { + final int m_prime = sequence.length; + if (m_prime == 0) { + return 0; + } + + final int pos_cost = (m_prime == 0) ? 0 : (32 - Integer.numberOfLeadingZeros(m_prime)); + double totalCost = 0; + + for (int val : sequence) { + if (val != 0) { + int absVal = Math.abs(val); + int val_cost = (absVal == 0) ? 0 : (32 - Integer.numberOfLeadingZeros(absVal)); + + totalCost += pos_cost + val_cost + 1; + } + } + + return totalCost; + } + + private byte[] performActualEncoding(double[] blockData, OptimalParams params, int n) + throws IOException { + logger.info( + "Performing actual encoding with beta={}, p_re={}, p_im={}", + params.betaStar, + params.pReStar, + params.pImStar); + + final FastFourierTransformer fft = new FastFourierTransformer(DftNormalization.STANDARD); + final Complex[] freqComplex = fft.transform(blockData, TransformType.FORWARD); + final int n_half = (n / 2) + 1; + final double quantizationFactor = Math.pow(2, params.betaStar); + + int[] F_hat_re = new int[n_half]; + int[] F_hat_im = new int[n_half]; + Complex[] F_hat_complex_dequant = new Complex[n]; + + for (int j = 0; j < n_half; j++) { + F_hat_re[j] = (int) Math.round(freqComplex[j].getReal() / quantizationFactor); + F_hat_im[j] = (int) Math.round(freqComplex[j].getImaginary() / quantizationFactor); + + F_hat_complex_dequant[j] = + new Complex( + (double) F_hat_re[j] * quantizationFactor, (double) F_hat_im[j] * quantizationFactor); + } + + for (int j = 1; j < n_half; j++) { + if (n - j < n) { + F_hat_complex_dequant[n - j] = F_hat_complex_dequant[j].conjugate(); + } + } + + final Complex[] reconstructedTimeComplex = + fft.transform(F_hat_complex_dequant, TransformType.INVERSE); + final int[] R_star = new int[n]; + for (int k = 0; k < n; k++) { + R_star[k] = (int) Math.round(blockData[k] - reconstructedTimeComplex[k].getReal()); + } + + params.dResidualStar = findOptimalDForResiduals(R_star); + logger.debug("Found optimal D* for residuals: {}", params.dResidualStar); + + ByteArrayOutputStream metadataBaos = new ByteArrayOutputStream(); + ByteArrayOutputStream freqBaos = new ByteArrayOutputStream(); + ByteArrayOutputStream residBaos = new ByteArrayOutputStream(); + + encodeMetadata(params, n, metadataBaos); + encodeFrequencyComponent(F_hat_re, F_hat_im, params.pReStar, params.pImStar, freqBaos); + encodeResidualComponent(R_star, params.dResidualStar, residBaos); + + ByteArrayOutputStream finalBaos = new ByteArrayOutputStream(); + finalBaos.write(metadataBaos.toByteArray()); + finalBaos.write(freqBaos.toByteArray()); + finalBaos.write(residBaos.toByteArray()); + + return finalBaos.toByteArray(); + } + + private void encodeMetadata(OptimalParams params, int n, ByteArrayOutputStream out) + throws IOException { + // IMPORTANT: 'n' must be encoded for the decoder. + + // 5 integers * 4 bytes/int = 20 bytes + ByteBuffer metaBuffer = ByteBuffer.allocate(20); + metaBuffer.putInt(n); // Add 'n' to the metadata + metaBuffer.putInt(params.betaStar); + metaBuffer.putInt(params.pReStar); + metaBuffer.putInt(params.pImStar); + metaBuffer.putInt(params.dResidualStar); + out.write(metaBuffer.array()); + logger.debug( + "Encoded metadata: n={}, beta={}, p_re={}, p_im={}, d_res={}", + n, + params.betaStar, + params.pReStar, + params.pImStar, + params.dResidualStar); + } + + private int findOptimalDForResiduals(int[] residuals) { + int bestD = 0; + double minCost = Double.MAX_VALUE; + + for (int d = 0; d < 16; d++) { + double currentCost = 0; + currentCost += (double) residuals.length * (d + 1); + + List<Integer> highBitValues = new ArrayList<>(); + int threshold = 1 << d; + for (int r : residuals) { + if (Math.abs(r) >= threshold) { + highBitValues.add(r >> d); + } + } + + currentCost += costDBP(highBitValues.stream().mapToInt(i -> i).toArray()); + + if (currentCost < minCost) { + minCost = currentCost; + bestD = d; + } + } + return bestD; + } + + private void encodeFrequencyComponent( + int[] F_re, int[] F_im, int p_re, int p_im, ByteArrayOutputStream out) throws IOException { + logger.debug("Encoding frequency components: p_re={}, p_im={}", p_re, p_im); + + encodeGVLE(Arrays.copyOfRange(F_re, 0, p_re), out); + encodeDBP(Arrays.copyOfRange(F_re, p_re, F_re.length), out); + + encodeGVLE(Arrays.copyOfRange(F_im, 0, p_im), out); + encodeDBP(Arrays.copyOfRange(F_im, p_im, F_im.length), out); + } + + private void encodeGVLE(int[] sequence, ByteArrayOutputStream out) throws IOException { + final int m = sequence.length; + if (m == 0) { + return; + } + logger.trace("Actual GVLE encoding for seq length {}", m); + + BitOutputStream bitOut = new BitOutputStream(out); + + int[] wSuffix = new int[m]; + wSuffix[m - 1] = + (sequence[m - 1] == 0) ? 0 : 32 - Integer.numberOfLeadingZeros(Math.abs(sequence[m - 1])); + for (int i = m - 2; i >= 0; i--) { + int currentWidth = + (sequence[i] == 0) ? 0 : 32 - Integer.numberOfLeadingZeros(Math.abs(sequence[i])); + wSuffix[i] = Math.max(currentWidth, wSuffix[i + 1]); + } + + int currentIdx = 0; + while (currentIdx < m) { + int sharedWidth = wSuffix[currentIdx]; + int groupStartIdx = currentIdx; + while (currentIdx < m && wSuffix[currentIdx] == sharedWidth) { + currentIdx++; + } + int groupLength = currentIdx - groupStartIdx; + + bitOut.write(sharedWidth, 5); + bitOut.write(groupLength, 11); + + if (sharedWidth > 0) { + for (int i = groupStartIdx; i < currentIdx; i++) { + int val = sequence[i]; + bitOut.write(val >= 0 ? 0 : 1, 1); + bitOut.write(Math.abs(val), sharedWidth); + } + } + } + + bitOut.flush(); + } + + private static class DBPValue { + int value; + int position; + + DBPValue(int value, int position) { + this.value = value; + this.position = position; + } + } + + private void encodeDBP(int[] sequence, ByteArrayOutputStream out) throws IOException { + final int m = sequence.length; + if (m == 0) { + return; + } + logger.trace("Actual DBP encoding for seq length {}", m); + + List<DBPValue> nonZeros = new ArrayList<>(); + for (int i = 0; i < m; i++) { + if (sequence[i] != 0) { + nonZeros.add(new DBPValue(sequence[i], i)); + } + } + + if (nonZeros.isEmpty()) { + return; + } + + nonZeros.sort((a, b) -> Integer.compare(Math.abs(b.value), Math.abs(a.value))); + + BitOutputStream bitOut = new BitOutputStream(out); + + int countWidth = (m == 0) ? 0 : 32 - Integer.numberOfLeadingZeros(m); + bitOut.write(nonZeros.size(), 16); + + int posWidth = (m == 0) ? 0 : 32 - Integer.numberOfLeadingZeros(m - 1); + if (posWidth == 0) posWidth = 1; + + for (DBPValue dbpVal : nonZeros) { + bitOut.write(dbpVal.position, posWidth); + } + + int firstVal = nonZeros.get(0).value; + int firstValAbs = Math.abs(firstVal); + int prevValWidth = (firstValAbs == 0) ? 1 : 32 - Integer.numberOfLeadingZeros(firstValAbs); + + bitOut.write(prevValWidth, 5); + + bitOut.write(firstVal >= 0 ? 0 : 1, 1); + bitOut.write(firstValAbs, prevValWidth); + + for (int i = 1; i < nonZeros.size(); i++) { + int currentVal = nonZeros.get(i).value; + int currentValAbs = Math.abs(currentVal); + + bitOut.write(currentVal >= 0 ? 0 : 1, 1); + bitOut.write(currentValAbs, prevValWidth); + + prevValWidth = (currentValAbs == 0) ? 1 : 32 - Integer.numberOfLeadingZeros(currentValAbs); + } + + bitOut.flush(); + } + + private void encodeResidualComponent(int[] residuals, int optimalD, ByteArrayOutputStream out) + throws IOException { + final int n = residuals.length; + if (n == 0) { + return; + } + logger.debug("Actual Hybrid Residual Encoding for {} residuals with optimalD={}", n, optimalD); + + BitOutputStream bitOut = new BitOutputStream(out); + + List<DBPValue> highBitNonZeros = new ArrayList<>(); + int lowBitMask = (1 << optimalD) - 1; + + for (int i = 0; i < n; i++) { + int r = residuals[i]; + int absR = Math.abs(r); + + bitOut.write(r >= 0 ? 0 : 1, 1); + + if (optimalD > 0) { + bitOut.write(absR & lowBitMask, optimalD); + } + + if (absR >= (1 << optimalD)) { + int highBitValue = absR >> optimalD; + int signedHighBitValue = (r < 0) ? -highBitValue : highBitValue; + + highBitNonZeros.add(new DBPValue(signedHighBitValue, i)); + } + } + + bitOut.flush(); + + ByteArrayOutputStream dbpBaos = new ByteArrayOutputStream(); + + int[] highBitSequence = new int[n]; + for (DBPValue dbpVal : highBitNonZeros) { + highBitSequence[dbpVal.position] = dbpVal.value; + } + + encodeDBP(highBitSequence, dbpBaos); + + out.write(dbpBaos.toByteArray()); + } + + @Override + public void encode(boolean value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(short value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(long value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(float value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(double value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(Binary value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public void encode(BigDecimal value, ByteArrayOutputStream out) { + throw new UnsupportedOperationException("FLEA only supports INT32."); + } + + @Override + public int getOneItemMaxSize() { + if (dataType == TSDataType.INT32) { + return 4 + 1; + } + throw new UnsupportedOperationException(dataType.toString()); + } + + @Override + public long getMaxByteSize() { + return (long) blockSize * 4 + 64; + } + + private static class BitOutputStream { + + private final ByteArrayOutputStream byteOutput; + private int bitBuffer; + private int bitsInBuffer; + + public BitOutputStream(ByteArrayOutputStream byteOutput) { + this.byteOutput = byteOutput; + this.bitBuffer = 0; + this.bitsInBuffer = 0; + } + + public void write(int value, int numBits) throws IOException { + if (numBits <= 0 || numBits > 32) { + throw new IllegalArgumentException("Number of bits must be between 1 and 32."); + } + + long mask = (1L << numBits) - 1; + bitBuffer = (bitBuffer << numBits) | ((int) value & (int) mask); + bitsInBuffer += numBits; + + while (bitsInBuffer >= 8) { + int byteToWrite = (bitBuffer >> (bitsInBuffer - 8)); + byteOutput.write(byteToWrite); + + bitsInBuffer -= 8; + } + } + + public void flush() throws IOException { + if (bitsInBuffer > 0) { + int byteToWrite = (bitBuffer << (8 - bitsInBuffer)); + byteOutput.write(byteToWrite); + + bitsInBuffer = 0; + bitBuffer = 0; + } + } + } +}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PeriodEncoder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PeriodEncoder.java deleted file mode 100644 index 10b094d..0000000 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/PeriodEncoder.java +++ /dev/null
@@ -1,582 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tsfile.encoding.encoder; - -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; - -import org.apache.commons.math3.complex.Complex; -import org.apache.commons.math3.transform.DftNormalization; -import org.apache.commons.math3.transform.FastFourierTransformer; -import org.apache.commons.math3.transform.TransformType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class PeriodEncoder extends Encoder { - - private static final Logger logger = LoggerFactory.getLogger(PeriodEncoder.class); - private TSDataType dataType; - private int maxStringLength; - - private static Object[] compRound(Complex[] dataf, int beta) { - Complex[] newDataf = new Complex[dataf.length]; - int[] ret = new int[dataf.length * 2]; - - for (int i = 0; i < dataf.length; i++) { - int a = (int) Math.round(dataf[i].getReal() / Math.pow(2, beta)); - int b = (int) Math.round(dataf[i].getImaginary() / Math.pow(2, beta)); - ret[2 * i] = a; - ret[2 * i + 1] = b; - - double newA = a * Math.pow(2, beta); - double newB = b * Math.pow(2, beta); - newDataf[i] = new Complex(newA, newB); - } - return new Object[] {newDataf, ret}; - } - - private static final FastFourierTransformer transformer = - new FastFourierTransformer(DftNormalization.STANDARD); - - private static Complex[] fft(double[] input) { - Complex[] transformed = transformer.transform(input, TransformType.FORWARD); - return transformed; - } - - private static double[] ifft(Complex[] input) { - - // 进行逆FFT，并除以长度进行归一化 - Complex[] inverse = transformer.transform(input, TransformType.INVERSE); - double[] result = new double[input.length]; - for (int i = 0; i < input.length; i++) { - result[i] = inverse[i].getReal() / input.length; - } - - return result; - } - - private static Complex[] rfft(double[] input) { - Complex[] transformed = transformer.transform(input, TransformType.FORWARD); - - // 只保留一半的结果（正频率部分） - int n = input.length; - Complex[] rfftResult = new Complex[n / 2 + 1]; - for (int i = 0; i < n / 2 + 1; i++) { - rfftResult[i] = transformed[i]; - } - - return rfftResult; - } - - private static double[] irfft(Complex[] input, int n) { - Complex[] conjugates = new Complex[n]; - - // 构造逆FFT所需的复数数组 - for (int i = 0; i < input.length; i++) { - if (i == 0 || (i == input.length - 1 && n % 2 == 0)) { - conjugates[i] = input[i]; - } else { - conjugates[i] = input[i]; - conjugates[n - i] = input[i].conjugate(); - } - } - - // 进行逆FFT，并除以长度进行归一化 - Complex[] inverse = transformer.transform(conjugates, TransformType.INVERSE); - double[] result = new double[n]; - for (int i = 0; i < n; i++) { - result[i] = inverse[i].getReal() / n; - } - - return result; - } - - // 计算多项式乘法 - private static double[] polyMul(double[] p1, double[] p2) { - int n = (int) Math.pow(2, Math.ceil(Math.log(p1.length + p2.length - 1) / Math.log(2))); - p1 = padArray(p1, n); - p2 = padArray(p2, n); - Complex[] p1Transformed = fft(p1); - Complex[] p2Transformed = fft(p2); - Complex[] resultTransformed = new Complex[n]; - for (int i = 0; i < n; i++) { - resultTransformed[i] = p1Transformed[i].multiply(p2Transformed[i]); - } - return ifft(resultTransformed); - } - - // 数组填充 - private static double[] padArray(double[] array, int length) { - double[] padded = new double[length]; - System.arraycopy(array, 0, padded, 0, array.length); - return padded; - } - - // 归一化 - private static double[] normalize(double[] x) { - double mean = mean(x); - double std = std(x); - double[] normalized = new double[x.length]; - for (int i = 0; i < x.length; i++) { - normalized[i] = (x[i] - mean) / std; - } - return normalized; - } - - // 计算均值 - private static double mean(double[] x) { - double sum = 0; - for (double v : x) { - sum += v; - } - return sum / x.length; - } - - // 计算标准差 - private static double std(double[] x) { - double mean = mean(x); - double sum = 0; - for (double v : x) { - sum += Math.pow(v - mean, 2); - } - return Math.sqrt(sum / x.length); - } - - // 用fft加速计算自相关函数 - private static double[] selfCorrFast(double[] x) { - x = normalize(x); - int N = x.length; - double[] xSelf = polyMul(x, reverseArray(x)); - double[] result = new double[(int) (N * (2.0 / 3.0))]; - for (int i = 0; i < result.length; i++) { - result[i] = xSelf[N - 1 - i] / (N - i); - } - return result; - } - - // 数组反转 - private static double[] reverseArray(double[] x) { - double[] reversed = new double[x.length]; - for (int i = 0; i < x.length; i++) { - reversed[i] = x[x.length - 1 - i]; - } - return reversed; - } - - // 给出数组中是邻近k个内最大的数的下标数组 - private static int[] pinkLocalMax(double[] data) { - final int k = 8; // 直接引入常数 - int[] result = new int[data.length]; - int count = 0; - for (int i = k; i < data.length - k; i++) { - double maxVal = data[i]; - for (int j = Math.max(0, i - k); j < Math.min(data.length, i + k); j++) { - maxVal = Math.max(maxVal, data[j]); - } - if (maxVal == data[i]) { - result[count++] = i; - } - } - return Arrays.copyOf(result, count); - } - - // 借助fft求出周期长度 - private static int getPeriod(double[] data) { - final double p = 0.5; // 直接引入常数 - double[] dataCorr = selfCorrFast(data); - int[] points = pinkLocalMax(dataCorr); - for (int point : points) { - if (point > 0 && dataCorr[point] > p) { - return point; - } - } - return 0; - } - - static class ByteOutToys { - private final ByteArrayOutputStream outputStream; - - public ByteOutToys(ByteArrayOutputStream outputStream) { - this.outputStream = outputStream; - } - - public void encode(int value, int bits) throws IOException { - for (int i = bits - 1; i >= 0; i--) { - outputStream.write((value >> i) & 1); - } - } - } - - private static final int MAX_SIZE = 0x7FFFFFFF; - private static final int MAX_VALUE = 0x7FFFFFFF; - private static final int GROUP_SIZE = 8; - - private static int bitLength(int value) { - return Integer.SIZE - Integer.numberOfLeadingZeros(value); - } - - private static int[] getCnt(int[] data) { - int[] cnt = new int[bitLength(MAX_VALUE) + 1]; - for (int i = 0; i < data.length; i++) { - if (data[i] != 0) { - cnt[bitLength(Math.abs(data[i]))]++; - } - } - return cnt; - } - - private static int[] bitLengthOrder(int[] data) { - int[] cnt = getCnt(data); - cnt[0] = 0; - for (int i = cnt.length - 2; i >= 0; i--) { - cnt[i] += cnt[i + 1]; - } - int n = cnt[0]; - int[] result = new int[n]; - for (int i = data.length - 1; i >= 0; i--) { - if (data[i] != 0) { - result[cnt[bitLength(Math.abs(data[i]))] - 1] = i; - cnt[bitLength(Math.abs(data[i]))]--; - } - } - return result; - } - - private static void descendingBitPacking(ByteOutToys stream, int[] data, boolean sgn) - throws IOException { - stream.encode(sgn ? 1 : 0, 1); - int[] index = bitLengthOrder(data); - stream.encode(data.length, bitLength(MAX_SIZE)); - stream.encode(index.length, bitLength(MAX_SIZE)); - if (index.length == 0) { - return; - } - for (int i = 0; i < index.length; i += GROUP_SIZE) { - int maxLen = 0; - for (int j = i; j < Math.min(i + GROUP_SIZE, index.length); j++) { - maxLen = Math.max(maxLen, bitLength(index[j])); - } - stream.encode(maxLen, bitLength(bitLength(MAX_SIZE))); - for (int j = i; j < Math.min(i + GROUP_SIZE, index.length); j++) { - stream.encode(index[j], maxLen); - } - } - int firstLen = bitLength(Math.abs(data[index[0]])); - int currentLen = firstLen; - stream.encode(firstLen, bitLength(bitLength(MAX_VALUE))); - for (int i : index) { - if (sgn) { - stream.encode(data[i] < 0 ? 1 : 0, 1); - } - stream.encode(Math.abs(data[i]), currentLen); - currentLen = bitLength(Math.abs(data[i])); - } - } - - private static int descendingBitPackingEstimate(int[] cnt, int n, boolean sgn) { - int sum1 = 0; - int sum2 = 0; - for (int i = 1; i < cnt.length; i++) { - sum1 += cnt[i]; - sum2 += cnt[i] * (i + (sgn ? 1 : 0)); - } - return bitLength(n) * sum1 + sum2; - } - - private static int calcSeparateStorageLength(int[] cnt, int n, int D) { - return n * (D + 1) - + descendingBitPackingEstimate(Arrays.copyOfRange(cnt, D, cnt.length), n, false); - } - - private static int[] separateStorageEstimate(int[] data) { - int[] cnt = getCnt(data); - int result = calcSeparateStorageLength(cnt, data.length, 0); - int D = 0; - for (int current_D = 1; current_D <= bitLength(MAX_VALUE); current_D++) { - int tmp = calcSeparateStorageLength(cnt, data.length, current_D); - if (tmp < result) { - result = tmp; - D = current_D; - } - } - return new int[] {result, D}; - } - - private static void separateStorage(ByteOutToys stream, int[] data) throws IOException { - int[] result = separateStorageEstimate(data); - int n = data.length; - int D = result[1]; - - stream.encode(n, bitLength(MAX_SIZE)); - stream.encode(D, bitLength(MAX_VALUE)); - - // low-bit part - for (int i : data) { - stream.encode(i < 0 ? 1 : 0, 1); // sgn bit - stream.encode(Math.abs(i) & ((1 << D) - 1), D); // low bits - } - - // high-bit part - int[] highBits = new int[n]; - for (int i = 0; i < n; i++) { - highBits[i] = Math.abs(data[i]) >> D; - } - descendingBitPacking(stream, highBits, false); - } - - public static int[] getRes(int[] data, Complex[] dataf, int p, int k) { - - double[] irfftResult = irfft(dataf, p); - - // 进行四舍五入并缩放到整数 - int[] rounded = new int[irfftResult.length]; - for (int i = 0; i < irfftResult.length; i++) { - rounded[i] = (int) Math.round(irfftResult[i] / k); - } - - // 复制 k 次并截取到与原始数据相同的长度 - int[] repeated = new int[data.length]; - for (int i = 0; i < data.length; i++) { - repeated[i] = rounded[i % rounded.length]; - } - - // 计算残差 - int[] res = new int[data.length]; - for (int i = 0; i < data.length; i++) { - res[i] = repeated[i] - data[i]; - } - - return res; - } - - private static Object[] getDatafAndRes(int[] data, Complex[] dataf, int p, int k, int beta) { - Object[] roundResult = compRound(dataf, beta); - Complex[] roundedDataf = (Complex[]) roundResult[0]; - int[] ret = (int[]) roundResult[1]; - - int[] res = getRes(data, roundedDataf, p, k); - - return new Object[] {roundedDataf, ret, res}; - } - - private static void encodeWithBeta( - ByteOutToys stream, int[] data, Complex[] dataf, int p, int k, int beta) throws IOException { - Object[] result = getDatafAndRes(data, dataf, p, k, beta); - Complex[] roundedDataf = (Complex[]) result[0]; - int[] ret = (int[]) result[1]; - int[] res = (int[]) result[2]; - - descendingBitPacking(stream, ret, true); - - int[] diffRes = new int[res.length]; - diffRes[0] = res[0]; - for (int i = 1; i < res.length; i++) { - diffRes[i] = res[i] - res[i - 1]; - } - - separateStorage(stream, diffRes); - } - - private static int encodeWithBetaEstimate(int[] data, Complex[] dataf, int p, int k, int beta) { - Object[] roundResult = compRound(dataf, beta); - Complex[] roundedDataf = (Complex[]) roundResult[0]; - int[] ret = (int[]) roundResult[1]; - - int maxLen = 0; - for (int x : ret) { - maxLen = Math.max(maxLen, bitLength(x)); - } - if (maxLen > bitLength(MAX_VALUE)) { - return -1; - } - - int result = descendingBitPackingEstimate(getCnt(ret), ret.length, true); - - int[] res = getRes(data, roundedDataf, p, k); - int[] diffRes = new int[res.length]; - diffRes[0] = res[0]; - for (int i = 1; i < res.length; i++) { - diffRes[i] = res[i] - res[i - 1]; - } - int[] separateResult = separateStorageEstimate(diffRes); - result += (int) separateResult[0]; - - return result; - } - - private static int getBeta(int[] data, Complex[] dataf, int p, int k) { - int result = encodeWithBetaEstimate(data, dataf, p, k, 0); - int beta = 0; - for (int currentBeta = -bitLength(MAX_VALUE) / 2; - currentBeta <= bitLength(MAX_VALUE); - currentBeta++) { - if (currentBeta != 0) { - int tmp = encodeWithBetaEstimate(data, dataf, p, k, currentBeta); - if (result == -1 || (tmp != -1 && tmp < result)) { - result = tmp; - beta = currentBeta; - } - } - } - return beta; - } - - private static void periodEncode(int[] data, ByteArrayOutputStream out) throws IOException { - ByteOutToys stream = new ByteOutToys(out); - double[] dataDouble = new double[data.length]; - for (int i = 0; i < data.length; i++) dataDouble[i] = (double) data[i]; - int p = getPeriod(dataDouble); - if (p == 0) { - stream.encode(p, bitLength(MAX_SIZE)); - separateStorage(stream, data); - } else { - int k = (data.length + p - 1) / p; - int[] dataFull; - if (data.length % p == 0) { - dataFull = data; - } else { - dataFull = new int[p * k]; - for (int i = 0; i < data.length; i++) dataFull[i] = data[i]; - for (int i = data.length; i < p * k; i++) dataFull[i] = data[i - p]; - } - double[] dataFullDouble = new double[dataFull.length]; - for (int i = 0; i < dataFull.length; i++) dataFullDouble[i] = (double) dataFull[i]; - Complex[] datafPre = rfft(dataFullDouble); - Complex[] dataf = new Complex[(datafPre.length + k - 1) / k]; - for (int i = 0; i < dataf.length; i++) dataf[i] = datafPre[i * k]; - int result = encodeWithBetaEstimate(data, dataf, p, k, 0); - int beta = getBeta(data, dataf, p, k); - stream.encode(p, bitLength(MAX_SIZE)); - stream.encode(beta >= 0 ? 0 : 1, 1); - stream.encode(Math.abs(beta), bitLength(bitLength(MAX_VALUE))); - encodeWithBeta(stream, data, dataf, p, k, beta); - } - return; - } - - List<Integer> data; - - public PeriodEncoder() { - super(TSEncoding.PERIOD); - data = new ArrayList<>(); - // this.dataType = dataType; - // this.maxStringLength = maxStringLength; - } - - // @Override - // public void encode(boolean value, ByteArrayOutputStream out) { - // if (value) { - // out.write(1); - // } else { - // out.write(0); - // } - // } - - // @Override - // public void encode(short value, ByteArrayOutputStream out) { - // out.write((value >> 8) & 0xFF); - // out.write(value & 0xFF); - // } - - @Override - public void encode(int value, ByteArrayOutputStream out) { - data.add(value); - // ReadWriteForEncodingUtils.writeVarInt(value, out); - } - - // @Override - // public void encode(long value, ByteArrayOutputStream out) { - // for (int i = 7; i >= 0; i--) { - // out.write((byte) (((value) >> (i * 8)) & 0xFF)); - // } - // } - - // @Override - // public void encode(float value, ByteArrayOutputStream out) { - // int floatInt = Float.floatToIntBits(value); - // out.write((floatInt >> 24) & 0xFF); - // out.write((floatInt >> 16) & 0xFF); - // out.write((floatInt >> 8) & 0xFF); - // out.write(floatInt & 0xFF); - // } - - // @Override - // public void encode(double value, ByteArrayOutputStream out) { - // encode(Double.doubleToLongBits(value), out); - // } - - // @Override - // public void encode(Binary value, ByteArrayOutputStream out) { - // try { - // // write the length of the bytes - // encode(value.getLength(), out); - // // write value - // out.write(value.getValues()); - // } catch (IOException e) { - // logger.error( - // "tsfile-encoding PlainEncoder: error occurs when encode Binary value {}", - // value, e); - // } - // } - - @Override - public void flush(ByteArrayOutputStream out) throws IOException { - periodEncode(data.stream().mapToInt(i -> i).toArray(), out); - } - - @Override - public int getOneItemMaxSize() { - switch (dataType) { - // case BOOLEAN: - // return 1; - case INT32: - return 4; - // case INT64: - // return 8; - // case FLOAT: - // return 4; - // case DOUBLE: - // return 8; - // case TEXT: - // // refer to encode(Binary,ByteArrayOutputStream) - // return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength; - default: - throw new UnsupportedOperationException(dataType.toString()); - } - } - - @Override - public long getMaxByteSize() { - return 0; - } - - // @Override - // public void encode(BigDecimal value, ByteArrayOutputStream out) { - // throw new TsFileEncodingException( - // "tsfile-encoding PlainEncoder: current version does not support BigDecimal - // value encoding"); - // } -}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java index b4febd2..fa5c6a4 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/TSEncodingBuilder.java
@@ -73,8 +73,8 @@ return new Freq(); case ZIGZAG: return new Zigzag(); - case PERIOD: - return new Period(); + case FLEA: + return new Flea(); default: throw new UnsupportedOperationException(type.toString()); } @@ -403,7 +403,7 @@ } /** for all TSDataType. */ - public static class Period extends TSEncodingBuilder { + public static class Flea extends TSEncodingBuilder { // private int maxStringLength = // TSFileDescriptor.getInstance().getConfig().getMaxStringLength(); @@ -411,9 +411,9 @@ public Encoder getEncoder(TSDataType type) { switch (type) { case INT32: - return new PeriodEncoder(); + return new FLEAEncoder(); default: - throw new UnSupportedDataTypeException("PERIOD doesn't support data type: " + type); + throw new UnSupportedDataTypeException("FLEA doesn't support data type: " + type); } }
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java index 4f9d29b..8658a3b 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSEncoding.java
@@ -30,7 +30,7 @@ GORILLA((byte) 8), ZIGZAG((byte) 9), FREQ((byte) 10), - PERIOD((byte) 14); + FLEA((byte) 14); private final byte type; @@ -73,7 +73,7 @@ case 10: return TSEncoding.FREQ; case 14: - return TSEncoding.PERIOD; + return TSEncoding.FLEA; default: throw new IllegalArgumentException("Invalid input: " + encoding); }