test/distributed/org/apache/cassandra/distributed/test/repair/ForceRepairTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.cassandra.distributed.test.repair;

import java.io.IOException;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.commons.lang3.ArrayUtils;
import org.junit.Test;

import com.carrotsearch.hppc.LongArrayList;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.lifecycle.View;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.QueryResults;
import org.apache.cassandra.distributed.api.SimpleQueryResult;
import org.apache.cassandra.distributed.shared.AssertUtils;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.FBUtilities;
import org.assertj.core.api.Assertions;

public class ForceRepairTest extends TestBaseImpl
{
    /**
     * Port of python dtest "repair_tests/incremental_repair_test.py::TestIncRepair::test_force" but extends to test
     * all types of repair.
     */
    @Test
    public void force() throws IOException
    {
        force(false);
    }

    @Test
    public void forceWithDifference() throws IOException
    {
        force(true);
    }

    private void force(boolean includeDifference) throws IOException
    {
        long nowInMicro = System.currentTimeMillis() * 1000;
        try (Cluster cluster = Cluster.build(3)
                                      .withConfig(c -> c.set("hinted_handoff_enabled", false)
                                                        .with(Feature.values()))
                                      .start())
        {
            init(cluster);
            cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k INT PRIMARY KEY, v INT)"));

            for (int i = 0; i < 10; i++)
                cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k,v) VALUES (?, ?) USING TIMESTAMP ?"), ConsistencyLevel.ALL, i, i, nowInMicro++);

            String downAddress = cluster.get(2).callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().getHostAddressAndPort());
            ClusterUtils.stopUnchecked(cluster.get(2));
            cluster.get(1).runOnInstance(() -> {
                InetAddressAndPort neighbor;
                try
                {
                    neighbor = InetAddressAndPort.getByName(downAddress);
                }
                catch (UnknownHostException e)
                {
                    throw new RuntimeException(e);
                }
                while (FailureDetector.instance.isAlive(neighbor))
                    Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
            });


            // repair should fail because node2 is down
            IInvokableInstance node1 = cluster.get(1);

            for (String[] args : Arrays.asList(new String[]{ "--full" },
                                               new String[]{ "--full", "--preview" },
                                               new String[]{ "--full", "--validate"}, // nothing should be in the repaired set, so shouldn't stream
                                               new String[]{ "--preview" }, // IR Preview
                                               new String[]{ "--validate"}, // nothing should be in the repaired set, so shouldn't stream
                                               new String[0])) // IR
            {
                if (includeDifference)
                    node1.executeInternal(withKeyspace("INSERT INTO %s.tbl (k,v) VALUES (?, ?) USING TIMESTAMP ?"), -1, -1, nowInMicro++); // each loop should have a different timestamp, causing a new difference

                try
                {
                    node1.nodetoolResult(ArrayUtils.addAll(new String[] {"repair", KEYSPACE}, args)).asserts().failure();
                    node1.nodetoolResult(ArrayUtils.addAll(new String[] {"repair", KEYSPACE, "--force"}, args)).asserts().success();

                    assertNoRepairedAt(cluster);
                }
                catch (Exception | Error e)
                {
                    // tag the error to include which args broke
                    e.addSuppressed(new AssertionError("Failure for args: " + Arrays.toString(args)));
                    throw e;
                }
            }

            if (includeDifference)
            {
                SimpleQueryResult expected = QueryResults.builder()
                                                         .row(-1, -1)
                                                         .build();
                for (IInvokableInstance node : Arrays.asList(node1, cluster.get(3)))
                {
                    SimpleQueryResult results = node.executeInternalWithResult(withKeyspace("SELECT * FROM %s.tbl WHERE k=?"), -1);
                    expected.reset();
                    AssertUtils.assertRows(results, expected);
                }
            }
        }
    }

    private static void assertNoRepairedAt(Cluster cluster)
    {
        List<long[]> repairedAt = getRepairedAt(cluster, KEYSPACE, "tbl");
        Assertions.assertThat(repairedAt).hasSize(cluster.size());
        for (int i = 0; i < repairedAt.size(); i++)
        {
            long[] array = repairedAt.get(i);
            if (array == null)
            {
                // ignore downed nodes
                Assertions.assertThat(cluster.get(i + 1).isShutdown()).isTrue();
                continue;
            }
            Assertions.assertThat(array).isNotEmpty();
            for (long a : array)
                Assertions.assertThat(a).describedAs("node%d had a repaired sstable", i + 1).isEqualTo(0);
        }
    }

    private static List<long[]> getRepairedAt(Cluster cluster, String keyspace, String table)
    {
        return cluster.stream().map(i -> {
            if (i.isShutdown())
                return null;

            return i.callOnInstance(() -> {
                TableMetadata meta = Schema.instance.getTableMetadata(keyspace, table);
                ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(meta.id);

                View view = cfs.getTracker().getView();
                LongArrayList list = new LongArrayList();
                for (SSTableReader sstable : view.liveSSTables())
                {
                    try
                    {
                        StatsMetadata metadata = sstable.getSSTableMetadata();
                        list.add(metadata.repairedAt);
                    }
                    catch (Exception e)
                    {
                        // ignore
                    }
                }
                return list.toArray();
            });
        }).collect(Collectors.toList());
    }
}