Fix that successive evolution may be overwritten
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java index 040ec83..5f4ea20 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TsFileMetadata.java
@@ -19,8 +19,11 @@ package org.apache.tsfile.file.metadata; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; +import java.util.stream.Collectors; import org.apache.tsfile.compatibility.DeserializeConfig; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.exception.encrypt.EncryptException; @@ -28,6 +31,7 @@ import org.apache.tsfile.file.metadata.evolution.SchemaEvolution; import org.apache.tsfile.file.metadata.evolution.SchemaEvolution.Builder; import org.apache.tsfile.utils.BloomFilter; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteForEncodingUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -49,7 +53,8 @@ private Map<String, TableSchema> tableSchemaMap; private Map<String, TableSchema> evolvedTableSchemaMap; private boolean hasTableSchemaMapCache; - private Map<String, String> tsFileProperties; + private List<Pair<String, String>> tsFileProperties; + private Map<String, String> tsFilePropertyMap; // offset of MetaMarker.SEPARATOR private long metaOffset; @@ -123,12 +128,16 @@ if (buffer.hasRemaining()) { int propertiesSize = ReadWriteForEncodingUtils.readVarInt(buffer); - Map<String, String> propertiesMap = new LinkedHashMap<>(); + List<Pair<String, String>> properties = new ArrayList<>(); + for (int i = 0; i < propertiesSize; i++) { String key = ReadWriteIOUtils.readVarIntString(buffer); String value = ReadWriteIOUtils.readVarIntString(buffer); - propertiesMap.put(key, value); + properties.add(new Pair<>(key, value)); } + + Map<String, String> propertiesMap = new HashMap<>(properties.size()); + properties.stream().forEach(p -> propertiesMap.put(p.getLeft(), p.getRight())); // if the file is not encrypted, set the default value(for compatible reason) if (!propertiesMap.containsKey("encryptLevel") || propertiesMap.get("encryptLevel") == null) { propertiesMap.put("encryptLevel", "0"); @@ -169,18 +178,19 @@ throw new EncryptException( "Unsupported encryptLevel: " + propertiesMap.get("encryptLevel")); } - fileMetaData.tsFileProperties = propertiesMap; + fileMetaData.tsFilePropertyMap = propertiesMap; + fileMetaData.tsFileProperties = properties; - fileMetaData.update(propertiesMap); + fileMetaData.update(properties); } return fileMetaData; } // update the TsFileMetadata according to schema evolutions defined in properties - private void update(Map<String, String> propertiesMap) { + private void update(List<Pair<String, String>> tsFileProperties) { Builder schemaEvolutionBuilder = new Builder(); - tsFileProperties.entrySet().forEach(entry -> { + tsFileProperties.forEach(entry -> { SchemaEvolution evolution = schemaEvolutionBuilder.fromProperty(entry); if (evolution != null) { evolution.applyTo(this); @@ -189,10 +199,15 @@ } public void addProperty(String key, String value) { - if (tsFileProperties == null) { - tsFileProperties = new LinkedHashMap<>(); + if (tsFilePropertyMap == null) { + tsFilePropertyMap = new LinkedHashMap<>(); } - tsFileProperties.put(key, value); + tsFilePropertyMap.put(key, value); + + if (tsFileProperties == null) { + tsFileProperties = new ArrayList<>(); + } + tsFileProperties.add(new Pair<>(key, value)); } public String getEncryptType() { @@ -259,9 +274,9 @@ ReadWriteForEncodingUtils.writeVarInt( tsFileProperties != null ? tsFileProperties.size() : 0, outputStream); if (tsFileProperties != null) { - for (Entry<String, String> entry : tsFileProperties.entrySet()) { - byteLen += ReadWriteIOUtils.writeVar(entry.getKey(), outputStream); - byteLen += ReadWriteIOUtils.writeVar(entry.getValue(), outputStream); + for (Pair<String, String> entry : tsFileProperties) { + byteLen += ReadWriteIOUtils.writeVar(entry.getLeft(), outputStream); + byteLen += ReadWriteIOUtils.writeVar(entry.getRight(), outputStream); } } @@ -319,6 +334,10 @@ } public Map<String, String> getTsFileProperties() { + return tsFilePropertyMap; + } + + public List<Pair<String, String>> getTsFilePropertyList() { return tsFileProperties; }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java index 8272164..e0b917a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/ColumnRename.java
@@ -22,6 +22,7 @@ import java.util.Map; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.utils.Pair; /** * A schema evolution operation that renames a column in a table schema. @@ -61,9 +62,9 @@ return nameAfter; } - public static ColumnRename fromProperty(Map.Entry<String, String> property) { - String key = property.getKey(); - String nameAfter = property.getValue(); + public static ColumnRename fromProperty(Pair<String, String> property) { + String key = property.getLeft(); + String nameAfter = property.getRight(); int i = KEY_PREFIX.length(); int tableNameLengthStart = i;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java index 74b2b77..99ef47d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/SchemaEvolution.java
@@ -21,6 +21,7 @@ import java.util.Map.Entry; import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.utils.Pair; /** * A schema evolution operation that can be applied to a TableSchemaMap. @@ -58,8 +59,8 @@ * @return the SchemaEvolution instance, or null if the property key does not match any known * schema evolution operation */ - public SchemaEvolution fromProperty(Entry<String, String> property) { - String key = property.getKey(); + public SchemaEvolution fromProperty(Pair<String, String> property) { + String key = property.getLeft(); if (!key.startsWith(KEY_PREFIX)) { return null;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java index 3f10835..4d6c65c 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TableRename.java
@@ -23,6 +23,7 @@ import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TsFileMetadata; +import org.apache.tsfile.utils.Pair; /** * A schema evolution operation that renames a table in a schema map. @@ -65,9 +66,9 @@ return nameAfter; } - public static TableRename fromProperty(Map.Entry<String, String> property) { - String key = property.getKey(); - String nameAfter = property.getValue(); + public static TableRename fromProperty(Pair<String, String> property) { + String key = property.getLeft(); + String nameAfter = property.getRight(); String nameBefore = key.substring(TableRename.KEY_PREFIX.length());
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriter.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriter.java index fb5be1f..65851d4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriter.java
@@ -33,6 +33,7 @@ import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TsFileMetadata; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.TsFileWriter; @@ -67,7 +68,7 @@ * @param newProperties the new properties to append * @throws IOException if an I/O error occurs */ - public void appendProperties(Map<String, String> newProperties) throws IOException { + public void appendProperties(List<Pair<String, String>> newProperties) throws IOException { // read TsFileMetadata and its position TsFileMetadata tsFileMetadata; long metadataOffset; @@ -78,8 +79,8 @@ // calculate the position of properties and write new properties to a buffer int propertiesOffset = tsFileMetadata.getPropertiesOffset(); - Map<String, String> mergedProperties = tsFileMetadata.getTsFileProperties(); - mergedProperties.putAll(newProperties); + List<Pair<String, String>> mergedProperties = tsFileMetadata.getTsFilePropertyList(); + mergedProperties.addAll(newProperties); PublicBAOS newPropertiesBuffer = new PublicBAOS(4096); DataOutputStream dataOutputStream = new DataOutputStream(newPropertiesBuffer); ReadWriteIOUtils.writeVar(mergedProperties, dataOutputStream); @@ -127,12 +128,12 @@ long start = System.currentTimeMillis(); for (String file : files) { TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(file); - Map<String, String> newProperties = new LinkedHashMap<>(); + List<Pair<String, String>> newProperties = new ArrayList<>(); for (int i = 0; i < evolutionNum; i++) { SchemaEvolution evolution = new ColumnRename("t1", "s" + i, "s" + (i + 1)); - newProperties.put( + newProperties.add(new Pair<>( evolution.propertyKey(), - evolution.propertyValue() + evolution.propertyValue()) ); } rewriter.appendProperties(newProperties);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java index 4c4261f..9328ea6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -178,6 +178,20 @@ return length; } + public static int writeVar(List<Pair<String, String>> propertyList, OutputStream stream) throws IOException { + if (propertyList == null) { + return ReadWriteForEncodingUtils.writeVarInt(NO_BYTE_TO_READ, stream); + } + + int length = 0; + length += ReadWriteForEncodingUtils.writeVarInt(propertyList.size(), stream); + for (Pair<String, String> entry : propertyList) { + length += writeVar(entry.getLeft(), stream); + length += writeVar(entry.getRight(), stream); + } + return length; + } + public static void write(List<Map<String, String>> maps, OutputStream stream) throws IOException { for (Map<String, String> map : maps) { write(map, stream);
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java index 90bf451..8934b99 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaEvolutionTest.java
@@ -33,7 +33,6 @@ import org.apache.tsfile.exception.read.ReadProcessException; import org.apache.tsfile.exception.write.NoMeasurementException; import org.apache.tsfile.exception.write.NoTableException; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -41,6 +40,7 @@ import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.read.v4.ITsFileReader; import org.apache.tsfile.read.v4.TsFileReaderBuilder; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -130,7 +130,7 @@ // rename t1 -> t4 TableRename tableRename = new TableRename("t1", "t4"); TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); - rewriter.appendProperties(Collections.singletonMap(tableRename.propertyKey(), tableRename.propertyValue())); + rewriter.appendProperties(Collections.singletonList(new Pair<>(tableRename.propertyKey(), tableRename.propertyValue()))); // Verify the table has been renamed try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { @@ -150,7 +150,7 @@ // rename t2 -> t1 tableRename = new TableRename("t2", "t1"); - rewriter.appendProperties(Collections.singletonMap(tableRename.propertyKey(), tableRename.propertyValue())); + rewriter.appendProperties(Collections.singletonList(new Pair<>(tableRename.propertyKey(), tableRename.propertyValue()))); // Verify the table has been renamed try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { @@ -191,12 +191,76 @@ } @Test + public void testSuccessiveRename() + throws IOException, ReadProcessException, NoTableException, NoMeasurementException { + // rename t1 -> t4 + TableRename tableRename = new TableRename("t1", "t4"); + TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); + rewriter.appendProperties(Collections.singletonList(new Pair<>(tableRename.propertyKey(), tableRename.propertyValue()))); + // rename t2 -> t1 + tableRename = new TableRename("t2", "t1"); + rewriter.appendProperties(Collections.singletonList(new Pair<>(tableRename.propertyKey(), tableRename.propertyValue()))); + // rename t1 -> t5 + tableRename = new TableRename("t1", "t5"); + rewriter.appendProperties(Collections.singletonList(new Pair<>(tableRename.propertyKey(), tableRename.propertyValue()))); + + // (t1, t2, t3) -> (t4, t5, t3) + // Verify the table has been renamed + try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { + assertFalse(reader.getTableSchemas("t1").isPresent()); + assertFalse(reader.getTableSchemas("t2").isPresent()); + Optional<TableSchema> t = reader.getTableSchemas("t4"); + assertTrue(t.isPresent()); + TableSchema tableSchema = t.get(); + assertEquals("t1", tableSchema.getTableName()); + t = reader.getTableSchemas("t5"); + assertTrue(t.isPresent()); + tableSchema = t.get(); + assertEquals("t2", tableSchema.getTableName()); + + assertThrows(NoTableException.class, () -> reader.query("t1", Arrays.asList("f1", "f2"), 0, 10)); + assertThrows(NoTableException.class, () -> reader.query("t2", Arrays.asList("f1", "f2"), 0, 10)); + ResultSet resultSet = reader.query("t4", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(3, resultSet.getInt("f1")); + assertEquals(4.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + resultSet = reader.query("t5", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(13, resultSet.getInt("f1")); + assertEquals(14.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + } + + // t3 is not affected + try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { + Optional<TableSchema> t1 = reader.getTableSchemas("t3"); + assertTrue(t1.isPresent()); + TableSchema tableSchema = t1.get(); + assertEquals("t3", tableSchema.getTableName()); + + ResultSet resultSet = reader.query("t3", Arrays.asList("f1", "f2"), 0, 10); + assertTrue(resultSet.next()); + assertEquals(23, resultSet.getInt("f1")); + assertEquals(24.0, resultSet.getDouble("f2"), 0.0001); + assertFalse(resultSet.next()); + } + + // test read timeseries metadata + try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(TEST_FILE_PATH)) { + TimeseriesMetadata timeseriesMetadata = sequenceReader.readTimeseriesMetadata( + Factory.DEFAULT_FACTORY.create(new String[]{"t5", "t2-tag1", "t2-tag2"}), "f1", false); + assertEquals(13, timeseriesMetadata.getStatistics().getMaxValue()); + } + } + + @Test public void testColumnRename() throws IOException, ReadProcessException, NoTableException, NoMeasurementException { // rename t1.f1 -> t1.f3 SchemaEvolution evolution = new ColumnRename("t1", "f1", "f3"); TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); - rewriter.appendProperties(Collections.singletonMap(evolution.propertyKey(), evolution.propertyValue())); + rewriter.appendProperties(Collections.singletonList(new Pair<>(evolution.propertyKey(), evolution.propertyValue()))); // Verify the column has been renamed try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) { @@ -217,7 +281,7 @@ // rename t1.f2 -> t1.f1 evolution = new ColumnRename("t1", "f2", "f1"); - rewriter.appendProperties(Collections.singletonMap(evolution.propertyKey(), evolution.propertyValue())); + rewriter.appendProperties(Collections.singletonList(new Pair<>(evolution.propertyKey(), evolution.propertyValue()))); // Verify the table has been renamed try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(TEST_FILE_PATH)).build()) {
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java index ee47fd1..a3bc23e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/evolution/TsFileSchemaRewriterTest.java
@@ -22,9 +22,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.TsFileMetadata; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.IntDataPoint; @@ -78,8 +83,8 @@ public void testAppendSingleProperty() throws IOException { // Create rewriter and append a single property TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); - Map<String, String> newProperties = new HashMap<>(); - newProperties.put("test_key", "test_value"); + List<Pair<String, String>> newProperties = new ArrayList<>(); + newProperties.add(new Pair<>("test_key", "test_value")); rewriter.appendProperties(newProperties); // Verify the property was added @@ -97,7 +102,8 @@ newProperties.put("key1", "value1"); newProperties.put("key2", "value2"); newProperties.put("key3", "value3"); - rewriter.appendProperties(newProperties); + rewriter.appendProperties(newProperties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); // Verify all properties were added try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { @@ -116,18 +122,21 @@ // First append Map<String, String> firstProperties = new HashMap<>(); firstProperties.put("key1", "value1"); - rewriter.appendProperties(firstProperties); + rewriter.appendProperties(firstProperties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); // Second append Map<String, String> secondProperties = new HashMap<>(); secondProperties.put("key2", "value2"); - rewriter.appendProperties(secondProperties); + rewriter.appendProperties(secondProperties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); // Third append with update to existing property Map<String, String> thirdProperties = new HashMap<>(); thirdProperties.put("key1", "new_value1"); thirdProperties.put("key3", "value3"); - rewriter.appendProperties(thirdProperties); + rewriter.appendProperties(thirdProperties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); // Verify final state try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { @@ -142,8 +151,7 @@ @Test public void testAppendEmptyProperties() throws IOException { TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(TEST_FILE_PATH); - Map<String, String> emptyProperties = new HashMap<>(); - rewriter.appendProperties(emptyProperties); + rewriter.appendProperties(Collections.emptyList()); // Verify file is still valid and only encryption-related properties were added try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) { @@ -161,7 +169,8 @@ TsFileSchemaRewriter rewriter = new TsFileSchemaRewriter(nonExistentFile); Map<String, String> properties = new HashMap<>(); properties.put("key", "value"); - rewriter.appendProperties(properties); + rewriter.appendProperties(properties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); } @Test @@ -178,7 +187,8 @@ largeProperties.put("key" + i, value.toString()); } - rewriter.appendProperties(largeProperties); + rewriter.appendProperties(largeProperties.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).collect( + Collectors.toList())); // Verify all properties were written correctly try (TsFileSequenceReader reader = new TsFileSequenceReader(TEST_FILE_PATH)) {