| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.cassandra.io.sstable; |
| |
| import java.io.File; |
| import java.nio.ByteBuffer; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.UUID; |
| import java.util.function.Function; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| import java.util.stream.Collectors; |
| import java.util.stream.IntStream; |
| import java.util.stream.Stream; |
| |
| import com.google.common.collect.Sets; |
| import org.junit.After; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| import org.apache.cassandra.config.CFMetaData; |
| import org.apache.cassandra.config.ColumnDefinition; |
| import org.apache.cassandra.config.DatabaseDescriptor; |
| import org.apache.cassandra.cql3.FieldIdentifier; |
| import org.apache.cassandra.cql3.statements.IndexTarget; |
| import org.apache.cassandra.db.SerializationHeader; |
| import org.apache.cassandra.db.marshal.AbstractCompositeType; |
| import org.apache.cassandra.db.marshal.AbstractType; |
| import org.apache.cassandra.db.marshal.CollectionType; |
| import org.apache.cassandra.db.marshal.CompositeType; |
| import org.apache.cassandra.db.marshal.FloatType; |
| import org.apache.cassandra.db.marshal.FrozenType; |
| import org.apache.cassandra.db.marshal.Int32Type; |
| import org.apache.cassandra.db.marshal.ListType; |
| import org.apache.cassandra.db.marshal.MapType; |
| import org.apache.cassandra.db.marshal.SetType; |
| import org.apache.cassandra.db.marshal.TupleType; |
| import org.apache.cassandra.db.marshal.UTF8Type; |
| import org.apache.cassandra.db.marshal.UserType; |
| import org.apache.cassandra.db.rows.EncodingStats; |
| import org.apache.cassandra.dht.Murmur3Partitioner; |
| import org.apache.cassandra.io.sstable.format.SSTableFormat; |
| import org.apache.cassandra.io.sstable.format.Version; |
| import org.apache.cassandra.io.sstable.format.big.BigFormat; |
| import org.apache.cassandra.io.sstable.metadata.MetadataType; |
| import org.apache.cassandra.io.util.FileUtils; |
| import org.apache.cassandra.io.util.SequentialWriter; |
| import org.apache.cassandra.schema.IndexMetadata; |
| import org.apache.cassandra.schema.SchemaKeyspace; |
| import org.apache.cassandra.utils.ByteBufferUtil; |
| import org.apache.cassandra.utils.FBUtilities; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertTrue; |
| import static org.junit.Assert.fail; |
| |
| /** |
| * Test the functionality of {@link SSTableHeaderFix}. |
| * It writes an 'big-m' version sstable(s) and executes against these. |
| */ |
| public class SSTableHeaderFixTest |
| { |
| static |
| { |
| DatabaseDescriptor.toolInitialization(); |
| DatabaseDescriptor.applyAddressConfig(); |
| } |
| |
| private File temporaryFolder; |
| |
| @Before |
| public void setup() |
| { |
| File f = FileUtils.createTempFile("SSTableUDTFixTest", ""); |
| f.delete(); |
| f.mkdirs(); |
| temporaryFolder = f; |
| } |
| |
| @After |
| public void teardown() |
| { |
| FileUtils.deleteRecursive(temporaryFolder); |
| } |
| |
| private static final AbstractType<?> udtPK = makeUDT("udt_pk"); |
| private static final AbstractType<?> udtCK = makeUDT("udt_ck"); |
| private static final AbstractType<?> udtStatic = makeUDT("udt_static"); |
| private static final AbstractType<?> udtRegular = makeUDT("udt_regular"); |
| private static final AbstractType<?> udtInner = makeUDT("udt_inner"); |
| private static final AbstractType<?> udtNested = new UserType("ks", |
| ByteBufferUtil.bytes("udt_nested"), |
| Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), |
| new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), |
| Arrays.asList(UTF8Type.instance, |
| udtInner), |
| true); |
| private static final AbstractType<?> tupleInTuple = makeTuple(makeTuple()); |
| private static final AbstractType<?> udtInTuple = makeTuple(udtInner); |
| private static final AbstractType<?> tupleInComposite = CompositeType.getInstance(UTF8Type.instance, makeTuple()); |
| private static final AbstractType<?> udtInComposite = CompositeType.getInstance(UTF8Type.instance, udtInner); |
| private static final AbstractType<?> udtInList = ListType.getInstance(udtInner, true); |
| private static final AbstractType<?> udtInSet = SetType.getInstance(udtInner, true); |
| private static final AbstractType<?> udtInMap = MapType.getInstance(UTF8Type.instance, udtInner, true); |
| private static final AbstractType<?> udtInFrozenList = ListType.getInstance(udtInner, false); |
| private static final AbstractType<?> udtInFrozenSet = SetType.getInstance(udtInner, false); |
| private static final AbstractType<?> udtInFrozenMap = MapType.getInstance(UTF8Type.instance, udtInner, false); |
| |
| private static AbstractType<?> makeUDT2(String udtName, boolean multiCell) |
| { |
| return new UserType("ks", |
| ByteBufferUtil.bytes(udtName), |
| Arrays.asList(new FieldIdentifier(ByteBufferUtil.bytes("a_field")), |
| new FieldIdentifier(ByteBufferUtil.bytes("a_udt"))), |
| Arrays.asList(UTF8Type.instance, |
| udtInner), |
| multiCell); |
| } |
| |
| private static AbstractType<?> makeUDT(String udtName) |
| { |
| return new UserType("ks", |
| ByteBufferUtil.bytes(udtName), |
| Collections.singletonList(new FieldIdentifier(ByteBufferUtil.bytes("a_field"))), |
| Collections.singletonList(UTF8Type.instance), |
| true); |
| } |
| |
| private static TupleType makeTuple() |
| { |
| return makeTuple(Int32Type.instance); |
| } |
| |
| private static TupleType makeTuple(AbstractType<?> second) |
| { |
| return new TupleType(Arrays.asList(UTF8Type.instance, |
| second)); |
| } |
| |
| private static TupleType makeTupleSimple() |
| { |
| // TODO this should create a non-frozen tuple type for the sake of handling a dropped, non-frozen UDT |
| return new TupleType(Collections.singletonList(UTF8Type.instance)); |
| } |
| |
| private static final Version version = BigFormat.instance.getVersion("mc"); |
| |
| private CFMetaData tableMetadata; |
| private final Set<String> updatedColumns = new HashSet<>(); |
| |
| private ColumnDefinition getColDef(String n) |
| { |
| return tableMetadata.getColumnDefinition(ByteBufferUtil.bytes(n)); |
| } |
| |
| /** |
| * Very basic test whether {@link SSTableHeaderFix} detect a type mismatch (regular_c 'int' vs 'float'). |
| */ |
| @Test |
| public void verifyTypeMismatchTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| File sstable = generateFakeSSTable(dir, 1); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| ColumnDefinition cd = getColDef("regular_c"); |
| tableMetadata.removeColumnDefinition(cd); |
| tableMetadata.addColumnDefinition(ColumnDefinition.regularDef("ks", "cf", "regular_c", FloatType.instance)); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertTrue(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| @Test |
| public void verifyTypeMatchTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| File sstable = buildFakeSSTable(dir, 1, cols, false); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertTrue(updatedColumns.isEmpty()); |
| assertFalse(headerFix.hasError()); |
| assertFalse(headerFix.hasChanges()); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| /** |
| * Simulates the case when an sstable contains a column not present in the schema, which can just be ignored. |
| */ |
| @Test |
| public void verifyWithUnknownColumnTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "solr_query", UTF8Type.instance)); |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| ColumnDefinition cd = getColDef("solr_query"); |
| tableMetadata.removeColumnDefinition(cd); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| |
| /** |
| * Simulates the case when an sstable contains a column not present in the table but as a target for an index. |
| * It can just be ignored. |
| */ |
| @Test |
| public void verifyWithIndexedUnknownColumnTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "solr_query", UTF8Type.instance)); |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| ColumnDefinition cd = getColDef("solr_query"); |
| tableMetadata.indexes(tableMetadata.getIndexes().with(IndexMetadata.fromSchemaMetadata("some search index", IndexMetadata.Kind.CUSTOM, Collections.singletonMap(IndexTarget.TARGET_OPTION_NAME, "solr_query")))); |
| tableMetadata.removeColumnDefinition(cd); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| |
| @Test |
| public void complexTypeMatchTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_tuple", tupleInTuple)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_nested", udtNested)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_tuple", udtInTuple)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_composite", tupleInComposite)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_composite", udtInComposite)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_list", udtInList)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_set", udtInSet)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_map", udtInMap)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_list", udtInFrozenList)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_set", udtInFrozenSet)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_map", udtInFrozenMap)); |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", |
| "udt_nested", "udt_in_composite", "udt_in_list", "udt_in_set", "udt_in_map"), updatedColumns); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| |
| @Test |
| public void complexTypeDroppedColumnsMatchTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_tuple", tupleInTuple)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_nested", udtNested)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_tuple", udtInTuple)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "tuple_in_composite", tupleInComposite)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_composite", udtInComposite)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_list", udtInList)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_set", udtInSet)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_map", udtInMap)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_list", udtInFrozenList)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_set", udtInFrozenSet)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "udt_in_frozen_map", udtInFrozenMap)); |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| for (String col : new String[]{"tuple_in_tuple", "udt_nested", "udt_in_tuple", |
| "tuple_in_composite", "udt_in_composite", |
| "udt_in_list", "udt_in_set", "udt_in_map", |
| "udt_in_frozen_list", "udt_in_frozen_set", "udt_in_frozen_map"}) |
| { |
| ColumnDefinition cd = getColDef(col); |
| tableMetadata.removeColumnDefinition(cd); |
| AbstractType<?> dropType = SchemaKeyspace.expandUserTypes(cd.type); |
| tableMetadata.recordColumnDrop(new ColumnDefinition(cd.ksName, cd.cfName, cd.name, dropType, cd.position(), cd.kind), FBUtilities.timestampMicros()); |
| } |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| assertEquals(Sets.newHashSet("pk", "ck", "regular_b", "static_b", "udt_nested"), updatedColumns); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| // do not check the inner types, as the inner types were not fixed in the serialization-header (test thing) |
| assertFrozenUdt(header, true, false); |
| } |
| |
| @Test |
| public void variousDroppedUserTypes() throws Exception |
| { |
| File dir = temporaryFolder; |
| |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| |
| ColSpec[] colSpecs = new ColSpec[] |
| { |
| // 'frozen<udt>' / live |
| new ColSpec("frozen_udt_as_frozen_udt_live", |
| makeUDT2("frozen_udt_as_frozen_udt_live", false), |
| makeUDT2("frozen_udt_as_frozen_udt_live", false), |
| false, |
| false), |
| // 'frozen<udt>' / live / as 'udt' |
| new ColSpec("frozen_udt_as_unfrozen_udt_live", |
| makeUDT2("frozen_udt_as_unfrozen_udt_live", false), |
| makeUDT2("frozen_udt_as_unfrozen_udt_live", true), |
| false, |
| true), |
| // 'frozen<udt>' / dropped |
| new ColSpec("frozen_udt_as_frozen_udt_dropped", |
| SchemaKeyspace.expandUserTypes(makeUDT2("frozen_udt_as_frozen_udt_dropped", true).freezeNestedMulticellTypes().freeze()), |
| makeUDT2("frozen_udt_as_frozen_udt_dropped", false), |
| makeUDT2("frozen_udt_as_frozen_udt_dropped", false), |
| true, |
| false), |
| // 'frozen<udt>' / dropped / as 'udt' |
| new ColSpec("frozen_udt_as_unfrozen_udt_dropped", |
| SchemaKeyspace.expandUserTypes(makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes().freeze()), |
| makeUDT2("frozen_udt_as_unfrozen_udt_dropped", true), |
| makeUDT2("frozen_udt_as_unfrozen_udt_dropped", false), |
| true, |
| true), |
| // 'udt' / live |
| new ColSpec("unfrozen_udt_as_unfrozen_udt_live", |
| makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), |
| makeUDT2("unfrozen_udt_as_unfrozen_udt_live", true), |
| false, |
| false), |
| // 'udt' / dropped |
| // TODO unable to test dropping a non-frozen UDT, as that requires an unfrozen tuple as well |
| // new ColSpec("unfrozen_udt_as_unfrozen_udt_dropped", |
| // SchemaKeyspace.expandUserTypes(makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true).freezeNestedMulticellTypes()), |
| // makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), |
| // makeUDT2("unfrozen_udt_as_unfrozen_udt_dropped", true), |
| // true, |
| // false), |
| // 'frozen<tuple>' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') |
| new ColSpec("frozen_tuple_as_frozen_tuple_live", |
| makeTupleSimple(), |
| makeTupleSimple(), |
| false, |
| false), |
| // 'frozen<tuple>' as 'TupleType(multiCell=false' (there is nothing like 'FrozenType(TupleType(') |
| new ColSpec("frozen_tuple_as_frozen_tuple_dropped", |
| makeTupleSimple(), |
| makeTupleSimple(), |
| true, |
| false) |
| }; |
| |
| Arrays.stream(colSpecs).forEach(c -> cols.add(ColumnDefinition.regularDef("ks", "cf", c.name, |
| // use the initial column type for the serialization header header. |
| c.preFix))); |
| |
| Map<String, ColSpec> colSpecMap = Arrays.stream(colSpecs).collect(Collectors.toMap(c -> c.name, c -> c)); |
| File sstable = buildFakeSSTable(dir, 1, cols, (s) -> s.map(c -> { |
| ColSpec cs = colSpecMap.get(c.name.toString()); |
| if (cs == null) |
| return c; |
| // update the column type in the schema to the "correct" one. |
| return new ColumnDefinition(c.ksName, c.cfName, c.name, cs.schema, c.position(), c.kind); |
| })); |
| |
| Arrays.stream(colSpecs) |
| .filter(c -> c.dropped) |
| .forEach(c -> { |
| ColumnDefinition cd = getColDef(c.name); |
| tableMetadata.removeColumnDefinition(cd); |
| tableMetadata.recordColumnDrop(cd, FBUtilities.timestampMicros()); |
| }); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| for (ColSpec colSpec : colSpecs) |
| { |
| AbstractType<?> hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); |
| assertEquals(colSpec.name, colSpec.preFix, hdrType); |
| assertEquals(colSpec.name, colSpec.preFix.isMultiCell(), hdrType.isMultiCell()); |
| } |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| // Verify that all columns to fix are in the updatedColumns set (paranoid, yet) |
| Arrays.stream(colSpecs) |
| .filter(c -> c.mustFix) |
| .forEach(c -> assertTrue("expect " + c.name + " to be updated, but was not (" + updatedColumns + ")", updatedColumns.contains(c.name))); |
| // Verify that the number of updated columns maches the expected number of columns to fix |
| assertEquals(Arrays.stream(colSpecs).filter(c -> c.mustFix).count(), updatedColumns.size()); |
| |
| header = readHeader(sstable); |
| for (ColSpec colSpec : colSpecs) |
| { |
| AbstractType<?> hdrType = header.getRegularColumns().get(ByteBufferUtil.bytes(colSpec.name)); |
| assertEquals(colSpec.name, colSpec.expect, hdrType); |
| assertEquals(colSpec.name, colSpec.expect.isMultiCell(), hdrType.isMultiCell()); |
| } |
| } |
| |
| static class ColSpec |
| { |
| final String name; |
| final AbstractType<?> schema; |
| final AbstractType<?> preFix; |
| final AbstractType<?> expect; |
| final boolean dropped; |
| final boolean mustFix; |
| |
| ColSpec(String name, AbstractType<?> schema, AbstractType<?> preFix, boolean dropped, boolean mustFix) |
| { |
| this(name, schema, preFix, schema, dropped, mustFix); |
| } |
| |
| ColSpec(String name, AbstractType<?> schema, AbstractType<?> preFix, AbstractType<?> expect, boolean dropped, boolean mustFix) |
| { |
| this.name = name; |
| this.schema = schema; |
| this.preFix = preFix; |
| this.expect = expect; |
| this.dropped = dropped; |
| this.mustFix = mustFix; |
| } |
| } |
| |
| @Test |
| public void verifyTypeMatchCompositeKeyTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk1", UTF8Type.instance, 0)); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk2", udtPK, 1)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| File sstable = buildFakeSSTable(dir, 1, cols, false); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertFalse(headerFix.hasChanges()); |
| assertTrue(updatedColumns.isEmpty()); |
| |
| // must not have re-written the stats-component |
| header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| @Test |
| public void compositePartitionKey() throws Exception |
| { |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk1", UTF8Type.instance, 0)); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk2", udtPK, 1)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| |
| File dir = temporaryFolder; |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertTrue(header.getKeyType() instanceof CompositeType); |
| CompositeType keyType = (CompositeType) header.getKeyType(); |
| assertEquals(Arrays.asList(UTF8Type.instance, udtPK), keyType.getComponents()); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| assertEquals(Sets.newHashSet("pk2", "ck", "regular_b", "static_b"), updatedColumns); |
| |
| header = readHeader(sstable); |
| assertTrue(header.getKeyType() instanceof CompositeType); |
| keyType = (CompositeType) header.getKeyType(); |
| assertEquals(Arrays.asList(UTF8Type.instance, udtPK.freeze()), keyType.getComponents()); |
| } |
| |
| @Test |
| public void compositeClusteringKey() throws Exception |
| { |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck1", Int32Type.instance, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck2", udtCK, 1)); |
| commonColumns(cols); |
| |
| File dir = temporaryFolder; |
| File sstable = buildFakeSSTable(dir, 1, cols, true); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertEquals(Arrays.asList(Int32Type.instance, udtCK), header.getClusteringTypes()); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertFalse(headerFix.hasError()); |
| assertTrue(headerFix.hasChanges()); |
| assertEquals(Sets.newHashSet("pk", "ck2", "regular_b", "static_b"), updatedColumns); |
| |
| header = readHeader(sstable); |
| assertEquals(Arrays.asList(Int32Type.instance, udtCK.freeze()), header.getClusteringTypes()); |
| } |
| |
| /** |
| * Check whether {@link SSTableHeaderFix} can operate on a single file. |
| */ |
| @Test |
| public void singleFileUDTFixTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| File sstable = generateFakeSSTable(dir, 1); |
| |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| |
| SSTableHeaderFix headerFix = builder().withPath(sstable.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertTrue(headerFix.hasChanges()); |
| assertFalse(headerFix.hasError()); |
| |
| header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| |
| /** |
| * Check whether {@link SSTableHeaderFix} can operate on a file in a directory. |
| */ |
| @Test |
| public void singleDirectoryUDTFixTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| List<File> sstables = IntStream.range(1, 11) |
| .mapToObj(g -> generateFakeSSTable(dir, g)) |
| .collect(Collectors.toList()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertTrue(headerFix.hasChanges()); |
| assertFalse(headerFix.hasError()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| } |
| |
| /** |
| * Check whether {@link SSTableHeaderFix} can operate multiple, single files. |
| */ |
| @Test |
| public void multipleFilesUDTFixTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| List<File> sstables = IntStream.range(1, 11) |
| .mapToObj(g -> generateFakeSSTable(dir, g)) |
| .collect(Collectors.toList()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| SSTableHeaderFix.Builder builder = builder(); |
| sstables.stream().map(File::toPath).forEach(builder::withPath); |
| SSTableHeaderFix headerFix = builder.build(); |
| headerFix.execute(); |
| |
| assertTrue(headerFix.hasChanges()); |
| assertFalse(headerFix.hasError()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| } |
| |
| /** |
| * Check whether {@link SSTableHeaderFix} can operate multiple files in a directory. |
| */ |
| @Test |
| public void multipleFilesInDirectoryUDTFixTest() throws Exception |
| { |
| File dir = temporaryFolder; |
| List<File> sstables = IntStream.range(1, 11) |
| .mapToObj(g -> generateFakeSSTable(dir, g)) |
| .collect(Collectors.toList()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, false, true); |
| } |
| |
| SSTableHeaderFix headerFix = builder().withPath(dir.toPath()) |
| .build(); |
| headerFix.execute(); |
| |
| assertTrue(headerFix.hasChanges()); |
| assertFalse(headerFix.hasError()); |
| |
| for (File sstable : sstables) |
| { |
| SerializationHeader.Component header = readHeader(sstable); |
| assertFrozenUdt(header, true, true); |
| } |
| } |
| |
| private static final Pattern p = Pattern.compile(".* Column '([^']+)' needs to be updated from type .*"); |
| |
| private SSTableHeaderFix.Builder builder() |
| { |
| updatedColumns.clear(); |
| return SSTableHeaderFix.builder() |
| .schemaCallback(() -> (desc) -> tableMetadata) |
| .info(ln -> { |
| System.out.println("INFO: " + ln); |
| Matcher m = p.matcher(ln); |
| if (m.matches()) |
| updatedColumns.add(m.group(1)); |
| }) |
| .warn(ln -> System.out.println("WARN: " + ln)) |
| .error(ln -> System.out.println("ERROR: " + ln)); |
| } |
| |
| private File generateFakeSSTable(File dir, int generation) |
| { |
| List<ColumnDefinition> cols = new ArrayList<>(); |
| cols.add(ColumnDefinition.partitionKeyDef("ks", "cf", "pk", udtPK, 0)); |
| cols.add(ColumnDefinition.clusteringDef("ks", "cf", "ck", udtCK, 0)); |
| commonColumns(cols); |
| return buildFakeSSTable(dir, generation, cols, true); |
| } |
| |
| private void commonColumns(List<ColumnDefinition> cols) |
| { |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_a", UTF8Type.instance)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_b", udtRegular)); |
| cols.add(ColumnDefinition.regularDef("ks", "cf", "regular_c", Int32Type.instance)); |
| cols.add(ColumnDefinition.staticDef("ks", "cf", "static_a", UTF8Type.instance)); |
| cols.add(ColumnDefinition.staticDef("ks", "cf", "static_b", udtStatic)); |
| cols.add(ColumnDefinition.staticDef("ks", "cf", "static_c", Int32Type.instance)); |
| } |
| |
| private File buildFakeSSTable(File dir, int generation, List<ColumnDefinition> cols, boolean freezeInSchema) |
| { |
| return buildFakeSSTable(dir, generation, cols, freezeInSchema |
| ? s -> s.map(c -> new ColumnDefinition(c.ksName, c.cfName, c.name, freezeUdt(c.type), c.position(), c.kind)) |
| : s -> s); |
| } |
| |
| private File buildFakeSSTable(File dir, int generation, List<ColumnDefinition> cols, Function<Stream<ColumnDefinition>, Stream<ColumnDefinition>> freezer) |
| { |
| CFMetaData headerMetadata = CFMetaData.create("ks", "cf", |
| UUID.randomUUID(), |
| false, true, false, false, false, |
| cols, |
| new Murmur3Partitioner()); |
| |
| List<ColumnDefinition> schemaCols = freezer.apply(cols.stream()).collect(Collectors.toList()); |
| tableMetadata = CFMetaData.create("ks", "cf", |
| UUID.randomUUID(), |
| false, true, false, false, false, |
| schemaCols, |
| new Murmur3Partitioner()); |
| |
| try |
| { |
| Descriptor desc = new Descriptor(version, dir, "ks", "cf", generation, SSTableFormat.Type.BIG, Component.DATA); |
| |
| // Just create the component files - we don't really need those. |
| for (Component component : requiredComponents) |
| assertTrue(new File(desc.filenameFor(component)).createNewFile()); |
| |
| AbstractType<?> partitionKey = headerMetadata.getKeyValidator(); |
| List<AbstractType<?>> clusteringKey = headerMetadata.clusteringColumns() |
| .stream() |
| .map(cd -> cd.type) |
| .collect(Collectors.toList()); |
| Map<ByteBuffer, AbstractType<?>> staticColumns = headerMetadata.allColumns() |
| .stream() |
| .filter(cd -> cd.kind == ColumnDefinition.Kind.STATIC) |
| .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); |
| Map<ByteBuffer, AbstractType<?>> regularColumns = headerMetadata.allColumns() |
| .stream() |
| .filter(cd -> cd.kind == ColumnDefinition.Kind.REGULAR) |
| .collect(Collectors.toMap(cd -> cd.name.bytes, cd -> cd.type, (a, b) -> a)); |
| |
| File statsFile = new File(desc.filenameFor(Component.STATS)); |
| SerializationHeader.Component header = SerializationHeader.Component.buildComponentForTools(partitionKey, |
| clusteringKey, |
| staticColumns, |
| regularColumns, |
| EncodingStats.NO_STATS); |
| |
| try (SequentialWriter out = new SequentialWriter(statsFile)) |
| { |
| desc.getMetadataSerializer().serialize(Collections.singletonMap(MetadataType.HEADER, header), out, version); |
| out.finish(); |
| } |
| |
| return new File(desc.filenameFor(Component.DATA)); |
| } |
| catch (Exception e) |
| { |
| throw new RuntimeException(e); |
| } |
| } |
| |
| private AbstractType<?> freezeUdt(AbstractType<?> type) |
| { |
| if (type instanceof CollectionType) |
| { |
| if (type.getClass() == ListType.class) |
| { |
| ListType<?> cHeader = (ListType<?>) type; |
| return ListType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); |
| } |
| else if (type.getClass() == SetType.class) |
| { |
| SetType<?> cHeader = (SetType<?>) type; |
| return SetType.getInstance(freezeUdt(cHeader.getElementsType()), cHeader.isMultiCell()); |
| } |
| else if (type.getClass() == MapType.class) |
| { |
| MapType<?, ?> cHeader = (MapType<?, ?>) type; |
| return MapType.getInstance(freezeUdt(cHeader.getKeysType()), freezeUdt(cHeader.getValuesType()), cHeader.isMultiCell()); |
| } |
| } |
| else if (type instanceof AbstractCompositeType) |
| { |
| if (type.getClass() == CompositeType.class) |
| { |
| CompositeType cHeader = (CompositeType) type; |
| return CompositeType.getInstance(cHeader.types.stream().map(this::freezeUdt).collect(Collectors.toList())); |
| } |
| } |
| else if (type instanceof TupleType) |
| { |
| if (type.getClass() == UserType.class) |
| { |
| UserType cHeader = (UserType) type; |
| cHeader = cHeader.freeze(); |
| return new UserType(cHeader.keyspace, cHeader.name, cHeader.fieldNames(), |
| cHeader.allTypes().stream().map(this::freezeUdt).collect(Collectors.toList()), |
| cHeader.isMultiCell()); |
| } |
| } |
| return type; |
| } |
| |
| private void assertFrozenUdt(SerializationHeader.Component header, boolean frozen, boolean checkInner) |
| { |
| AbstractType<?> keyType = header.getKeyType(); |
| if (keyType instanceof CompositeType) |
| { |
| for (AbstractType<?> component : ((CompositeType) keyType).types) |
| assertFrozenUdt("partition-key-component", component, frozen, checkInner); |
| } |
| assertFrozenUdt("partition-key", keyType, frozen, checkInner); |
| |
| for (AbstractType<?> type : header.getClusteringTypes()) |
| assertFrozenUdt("clustering-part", type, frozen, checkInner); |
| for (Map.Entry<ByteBuffer, AbstractType<?>> col : header.getStaticColumns().entrySet()) |
| assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); |
| for (Map.Entry<ByteBuffer, AbstractType<?>> col : header.getRegularColumns().entrySet()) |
| assertFrozenUdt(UTF8Type.instance.compose(col.getKey()), col.getValue(), frozen, checkInner); |
| } |
| |
| private void assertFrozenUdt(String name, AbstractType<?> type, boolean frozen, boolean checkInner) |
| { |
| if (type instanceof CompositeType) |
| { |
| if (checkInner) |
| for (AbstractType<?> component : ((CompositeType) type).types) |
| assertFrozenUdt(name, component, frozen, true); |
| } |
| else if (type instanceof CollectionType) |
| { |
| if (checkInner) |
| { |
| if (type instanceof MapType) |
| { |
| MapType map = (MapType) type; |
| // only descend for non-frozen types (checking frozen in frozen is just stupid) |
| if (map.isMultiCell()) |
| { |
| assertFrozenUdt(name + "<map-key>", map.getKeysType(), frozen, true); |
| assertFrozenUdt(name + "<map-value>", map.getValuesType(), frozen, true); |
| } |
| } |
| else if (type instanceof SetType) |
| { |
| SetType set = (SetType) type; |
| // only descend for non-frozen types (checking frozen in frozen is just stupid) |
| if (set.isMultiCell()) |
| assertFrozenUdt(name + "<set>", set.getElementsType(), frozen, true); |
| } |
| else if (type instanceof ListType) |
| { |
| ListType list = (ListType) type; |
| // only descend for non-frozen types (checking frozen in frozen is just stupid) |
| if (list.isMultiCell()) |
| assertFrozenUdt(name + "<list>", list.getElementsType(), frozen, true); |
| } |
| } |
| } |
| else if (type instanceof TupleType) |
| { |
| if (checkInner) |
| { |
| TupleType tuple = (TupleType) type; |
| // only descend for non-frozen types (checking frozen in frozen is just stupid) |
| if (tuple.isMultiCell()) |
| for (AbstractType<?> component : tuple.allTypes()) |
| assertFrozenUdt(name + "<tuple>", component, frozen, true); |
| } |
| } |
| |
| if (type instanceof UserType) |
| { |
| String typeString = type.toString(); |
| assertEquals(name + ": " + typeString, frozen, !type.isMultiCell()); |
| if (typeString.startsWith(UserType.class.getName() + '(')) |
| if (frozen) |
| fail(name + ": " + typeString); |
| if (typeString.startsWith(FrozenType.class.getName() + '(' + UserType.class.getName() + '(')) |
| if (!frozen) |
| fail(name + ": " + typeString); |
| } |
| } |
| |
| private SerializationHeader.Component readHeader(File sstable) throws Exception |
| { |
| Descriptor desc = Descriptor.fromFilename(sstable.getParentFile(), sstable.getName()).left; |
| return (SerializationHeader.Component) desc.getMetadataSerializer().deserialize(desc, MetadataType.HEADER); |
| } |
| |
| private static final Component[] requiredComponents = new Component[]{ Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.TOC }; |
| } |
