/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.cassandra.cql3.validation.entities;

import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;

import org.apache.commons.lang3.StringUtils;
import org.junit.Test;

import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
import org.apache.cassandra.cql3.statements.IndexTarget;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.SyntaxException;
import org.apache.cassandra.index.IndexNotAvailableException;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.index.StubIndex;
import org.apache.cassandra.index.internal.CustomCassandraIndex;
import org.apache.cassandra.schema.IndexMetadata;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.MD5Digest;
import org.apache.cassandra.utils.Pair;

import static org.apache.cassandra.Util.throwAssert;
import static org.apache.cassandra.utils.ByteBufferUtil.EMPTY_BYTE_BUFFER;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class SecondaryIndexTest extends CQLTester
{
    private static final int TOO_BIG = 1024 * 65;

    @Test
    public void testCreateAndDropIndex() throws Throwable
    {
        testCreateAndDropIndex("test", false);
        testCreateAndDropIndex("test2", true);
    }

    @Test
    public void testCreateAndDropIndexWithQuotedIdentifier() throws Throwable
    {
        testCreateAndDropIndex("\"quoted_ident\"", false);
        testCreateAndDropIndex("\"quoted_ident2\"", true);
    }

    @Test
    public void testCreateAndDropIndexWithCamelCaseIdentifier() throws Throwable
    {
        testCreateAndDropIndex("CamelCase", false);
        testCreateAndDropIndex("CamelCase2", true);
    }

    /**
     * Test creating and dropping an index with the specified name.
     *
     * @param indexName         the index name
     * @param addKeyspaceOnDrop add the keyspace name in the drop statement
     * @throws Throwable if an error occurs
     */
    private void testCreateAndDropIndex(String indexName, boolean addKeyspaceOnDrop) throws Throwable
    {
        execute("USE system");
        assertInvalidMessage(String.format("Index '%s' could not be found",
                                           removeQuotes(indexName.toLowerCase(Locale.US))),
                             "DROP INDEX " + indexName + ";");

        createTable("CREATE TABLE %s (a int primary key, b int);");
        createIndex("CREATE INDEX " + indexName + " ON %s(b);");
        createIndex("CREATE INDEX IF NOT EXISTS " + indexName + " ON %s(b);");

        assertInvalidMessage(String.format("Index %s already exists",
                                           removeQuotes(indexName.toLowerCase(Locale.US))),
                             "CREATE INDEX " + indexName + " ON %s(b)");

        // IF NOT EXISTS should apply in cases where the new index differs from an existing one in name only
        String otherIndexName = "index_" + System.nanoTime();
        assertEquals(1, getCurrentColumnFamilyStore().metadata.getIndexes().size());
        createIndex("CREATE INDEX IF NOT EXISTS " + otherIndexName + " ON %s(b)");
        assertEquals(1, getCurrentColumnFamilyStore().metadata.getIndexes().size());
        assertInvalidMessage(String.format("Index %s is a duplicate of existing index %s",
                                           removeQuotes(otherIndexName.toLowerCase(Locale.US)),
                                           removeQuotes(indexName.toLowerCase(Locale.US))),
                             "CREATE INDEX " + otherIndexName + " ON %s(b)");

        execute("INSERT INTO %s (a, b) values (?, ?);", 0, 0);
        execute("INSERT INTO %s (a, b) values (?, ?);", 1, 1);
        execute("INSERT INTO %s (a, b) values (?, ?);", 2, 2);
        execute("INSERT INTO %s (a, b) values (?, ?);", 3, 1);

        assertRows(execute("SELECT * FROM %s where b = ?", 1), row(1, 1), row(3, 1));
        assertInvalidMessage(String.format("Index '%s' could not be found in any of the tables of keyspace 'system'",
                                           removeQuotes(indexName.toLowerCase(Locale.US))),
                             "DROP INDEX " + indexName);

        if (addKeyspaceOnDrop)
        {
            dropIndex("DROP INDEX " + KEYSPACE + "." + indexName);
        }
        else
        {
            execute("USE " + KEYSPACE);
            execute("DROP INDEX " + indexName);
        }

        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                             "SELECT * FROM %s where b = ?", 1);
        dropIndex("DROP INDEX IF EXISTS " + indexName);
        assertInvalidMessage(String.format("Index '%s' could not be found",
                                           removeQuotes(indexName.toLowerCase(Locale.US))),
                             "DROP INDEX " + indexName);
    }

    /**
     * Removes the quotes from the specified index name.
     *
     * @param indexName the index name from which the quotes must be removed.
     * @return the unquoted index name.
     */
    private static String removeQuotes(String indexName)
    {
        return StringUtils.remove(indexName, '\"');
    }

    /**
     * Check that you can query for an indexed column even with a key EQ clause,
     * migrated from cql_tests.py:TestCQL.static_cf_test()
     */
    @Test
    public void testSelectWithEQ() throws Throwable
    {
        createTable("CREATE TABLE %s (userid uuid PRIMARY KEY, firstname text, lastname text, age int)");
        createIndex("CREATE INDEX byAge ON %s(age)");

        UUID id1 = UUID.fromString("550e8400-e29b-41d4-a716-446655440000");
        UUID id2 = UUID.fromString("f47ac10b-58cc-4372-a567-0e02b2c3d479");

        execute("INSERT INTO %s (userid, firstname, lastname, age) VALUES (?, 'Frodo', 'Baggins', 32)", id1);
        execute("UPDATE %s SET firstname = 'Samwise', lastname = 'Gamgee', age = 33 WHERE userid = ?", id2);

        assertEmpty(execute("SELECT firstname FROM %s WHERE userid = ? AND age = 33", id1));

        assertRows(execute("SELECT firstname FROM %s WHERE userid = ? AND age = 33", id2),
                   row("Samwise"));
    }

    /**
     * Check CREATE INDEX without name and validate the index can be dropped,
     * migrated from cql_tests.py:TestCQL.nameless_index_test()
     */
    @Test
    public void testNamelessIndex() throws Throwable
    {
        createTable(" CREATE TABLE %s (id text PRIMARY KEY, birth_year int)");

        createIndex("CREATE INDEX on %s (birth_year)");

        execute("INSERT INTO %s (id, birth_year) VALUES ('Tom', 42)");
        execute("INSERT INTO %s (id, birth_year) VALUES ('Paul', 24)");
        execute("INSERT INTO %s (id, birth_year) VALUES ('Bob', 42)");

        assertRows(execute("SELECT id FROM %s WHERE birth_year = 42"),
                   row("Tom"),
                   row("Bob"));

        execute("DROP INDEX %s_birth_year_idx");

        assertInvalid("SELECT id FROM users WHERE birth_year = 42");
    }

    /**
     * Test range queries with 2ndary indexes (#4257),
     * migrated from cql_tests.py:TestCQL.range_query_2ndary_test()
     */
    @Test
    public void testRangeQuery() throws Throwable
    {
        createTable("CREATE TABLE %s (id int primary key, row int, setid int)");
        createIndex("CREATE INDEX indextest_setid_idx ON %s (setid)");

        execute("INSERT INTO %s (id, row, setid) VALUES (?, ?, ?)", 0, 0, 0);
        execute("INSERT INTO %s (id, row, setid) VALUES (?, ?, ?)", 1, 1, 0);
        execute("INSERT INTO %s (id, row, setid) VALUES (?, ?, ?)", 2, 2, 0);
        execute("INSERT INTO %s (id, row, setid) VALUES (?, ?, ?)", 3, 3, 0);

        assertInvalid("SELECT * FROM %s WHERE setid = 0 AND row < 1");

        assertRows(execute("SELECT * FROM %s WHERE setid = 0 AND row < 1 ALLOW FILTERING"),
                   row(0, 0, 0));
    }

    /**
     * Check for unknown compression parameters options (#4266),
     * migrated from cql_tests.py:TestCQL.compression_option_validation_test()
     */
    @Test
    public void testUnknownCompressionOptions() throws Throwable
    {
        String tableName = createTableName();
        assertInvalidThrow(SyntaxException.class, String.format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression_parameters:sstable_compressor = 'DeflateCompressor'", tableName));

        assertInvalidThrow(ConfigurationException.class, String.format("CREATE TABLE %s (key varchar PRIMARY KEY, password varchar, gender varchar) WITH compression = { 'sstable_compressor': 'DeflateCompressor' }",
                                                                       tableName));
    }

    /**
     * Check one can use arbitrary name for datacenter when creating keyspace (#4278),
     * migrated from cql_tests.py:TestCQL.keyspace_creation_options_test()
     */
    @Test
    public void testDataCenterName() throws Throwable
    {
       execute("CREATE KEYSPACE Foo WITH replication = { 'class' : 'NetworkTopologyStrategy', 'us-east' : 1, 'us-west' : 1 };");
    }

    /**
     * Migrated from cql_tests.py:TestCQL.indexes_composite_test()
     */
    @Test
    public void testIndexOnComposite() throws Throwable
    {
        String tableName = createTable("CREATE TABLE %s (blog_id int, timestamp int, author text, content text, PRIMARY KEY (blog_id, timestamp))");

        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 0, "bob", "1st post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 1, "tom", "2nd post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 2, "bob", "3rd post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 3, "tom", "4th post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 1, 0, "bob", "5th post");

        createIndex("CREATE INDEX authoridx ON %s (author)");

        assertTrue(waitForIndex(keyspace(), tableName, "authoridx"));

        assertRows(execute("SELECT blog_id, timestamp FROM %s WHERE author = 'bob'"),
                   row(1, 0),
                   row(0, 0),
                   row(0, 2));

        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 1, 1, "tom", "6th post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 1, 2, "tom", "7th post");
        execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 1, 3, "bob", "8th post");

        assertRows(execute("SELECT blog_id, timestamp FROM %s WHERE author = 'bob'"),
                   row(1, 0),
                   row(1, 3),
                   row(0, 0),
                   row(0, 2));

        execute("DELETE FROM %s WHERE blog_id = 0 AND timestamp = 2");

        assertRows(execute("SELECT blog_id, timestamp FROM %s WHERE author = 'bob'"),
                   row(1, 0),
                   row(1, 3),
                   row(0, 0));
    }

    /**
     * Test for the validation bug of #4709,
     * migrated from cql_tests.py:TestCQL.refuse_in_with_indexes_test()
     */
    @Test
    public void testInvalidIndexSelect() throws Throwable
    {
        createTable("create table %s (pk varchar primary key, col1 varchar, col2 varchar)");
        createIndex("create index on %s (col1)");
        createIndex("create index on %s (col2)");

        execute("insert into %s (pk, col1, col2) values ('pk1','foo1','bar1')");
        execute("insert into %s (pk, col1, col2) values ('pk1a','foo1','bar1')");
        execute("insert into %s (pk, col1, col2) values ('pk1b','foo1','bar1')");
        execute("insert into %s (pk, col1, col2) values ('pk1c','foo1','bar1')");
        execute("insert into %s (pk, col1, col2) values ('pk2','foo2','bar2')");
        execute("insert into %s (pk, col1, col2) values ('pk3','foo3','bar3')");
        assertInvalid("select * from %s where col2 in ('bar1', 'bar2')");

        //Migrated from cql_tests.py:TestCQL.bug_6050_test()
        createTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)");

        createIndex("CREATE INDEX ON %s (a)");
        assertInvalid("SELECT * FROM %s WHERE a = 3 AND b IN (1, 3)");

    }

    /**
     * Migrated from cql_tests.py:TestCQL.edge_2i_on_complex_pk_test()
     */
    @Test
    public void testIndexesOnComplexPrimaryKey() throws Throwable
    {
        createTable("CREATE TABLE %s (pk0 int, pk1 int, ck0 int, ck1 int, ck2 int, value int, PRIMARY KEY ((pk0, pk1), ck0, ck1, ck2))");

        execute("CREATE INDEX ON %s (pk0)");
        execute("CREATE INDEX ON %s (ck0)");
        execute("CREATE INDEX ON %s (ck1)");
        execute("CREATE INDEX ON %s (ck2)");

        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (0, 1, 2, 3, 4, 5)");
        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (1, 2, 3, 4, 5, 0)");
        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (2, 3, 4, 5, 0, 1)");
        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (3, 4, 5, 0, 1, 2)");
        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (4, 5, 0, 1, 2, 3)");
        execute("INSERT INTO %s (pk0, pk1, ck0, ck1, ck2, value) VALUES (5, 0, 1, 2, 3, 4)");

        assertRows(execute("SELECT value FROM %s WHERE pk0 = 2"),
                   row(1));

        assertRows(execute("SELECT value FROM %s WHERE ck0 = 0"),
                   row(3));

        assertRows(execute("SELECT value FROM %s WHERE pk0 = 3 AND pk1 = 4 AND ck1 = 0"),
                   row(2));

        assertRows(execute("SELECT value FROM %s WHERE pk0 = 5 AND pk1 = 0 AND ck0 = 1 AND ck2 = 3 ALLOW FILTERING"),
                   row(4));
    }

    /**
     * Test for CASSANDRA-5240,
     * migrated from cql_tests.py:TestCQL.bug_5240_test()
     */
    @Test
    public void testIndexOnCompoundRowKey() throws Throwable
    {
        createTable("CREATE TABLE %s (interval text, seq int, id int, severity int, PRIMARY KEY ((interval, seq), id) ) WITH CLUSTERING ORDER BY (id DESC)");

        execute("CREATE INDEX ON %s (severity)");

        execute("insert into %s (interval, seq, id , severity) values('t',1, 1, 1)");
        execute("insert into %s (interval, seq, id , severity) values('t',1, 2, 1)");
        execute("insert into %s (interval, seq, id , severity) values('t',1, 3, 2)");
        execute("insert into %s (interval, seq, id , severity) values('t',1, 4, 3)");
        execute("insert into %s (interval, seq, id , severity) values('t',2, 1, 3)");
        execute("insert into %s (interval, seq, id , severity) values('t',2, 2, 3)");
        execute("insert into %s (interval, seq, id , severity) values('t',2, 3, 1)");
        execute("insert into %s (interval, seq, id , severity) values('t',2, 4, 2)");

        assertRows(execute("select * from %s where severity = 3 and interval = 't' and seq =1"),
                   row("t", 1, 4, 3));
    }

    /**
     * Migrated from cql_tests.py:TestCQL.secondary_index_counters()
     */
    @Test
    public void testIndexOnCountersInvalid() throws Throwable
    {
        createTable("CREATE TABLE %s (k int PRIMARY KEY, c counter)");
        assertInvalid("CREATE INDEX ON test(c)");
    }

    /**
     * Migrated from cql_tests.py:TestCQL.collection_indexing_test()
     */
    @Test
    public void testIndexOnCollections() throws Throwable
    {
        createTable(" CREATE TABLE %s ( k int, v int, l list<int>, s set<text>, m map<text, int>, PRIMARY KEY (k, v))");

        createIndex("CREATE INDEX ON %s (l)");
        createIndex("CREATE INDEX ON %s (s)");
        createIndex("CREATE INDEX ON %s (m)");

        execute("INSERT INTO %s (k, v, l, s, m) VALUES (0, 0, [1, 2],    {'a'},      {'a' : 1})");
        execute("INSERT INTO %s (k, v, l, s, m) VALUES (0, 1, [3, 4],    {'b', 'c'}, {'a' : 1, 'b' : 2})");
        execute("INSERT INTO %s (k, v, l, s, m) VALUES (0, 2, [1],       {'a', 'c'}, {'c' : 3})");
        execute("INSERT INTO %s (k, v, l, s, m) VALUES (1, 0, [1, 2, 4], {},         {'b' : 1})");
        execute("INSERT INTO %s (k, v, l, s, m) VALUES (1, 1, [4, 5],    {'d'},      {'a' : 1, 'b' : 3})");

        // lists
        assertRows(execute("SELECT k, v FROM %s WHERE l CONTAINS 1"), row(1, 0), row(0, 0), row(0, 2));
        assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND l CONTAINS 1"), row(0, 0), row(0, 2));
        assertRows(execute("SELECT k, v FROM %s WHERE l CONTAINS 2"), row(1, 0), row(0, 0));
        assertEmpty(execute("SELECT k, v FROM %s WHERE l CONTAINS 6"));

        // sets
        assertRows(execute("SELECT k, v FROM %s WHERE s CONTAINS 'a'"), row(0, 0), row(0, 2));
        assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND s CONTAINS 'a'"), row(0, 0), row(0, 2));
        assertRows(execute("SELECT k, v FROM %s WHERE s CONTAINS 'd'"), row(1, 1));
        assertEmpty(execute("SELECT k, v FROM %s  WHERE s CONTAINS 'e'"));

        // maps
        assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS 1"), row(1, 0), row(1, 1), row(0, 0), row(0, 1));
        assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m CONTAINS 1"), row(0, 0), row(0, 1));
        assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS 2"), row(0, 1));
        assertEmpty(execute("SELECT k, v FROM %s  WHERE m CONTAINS 4"));
    }

    @Test
    public void testSelectOnMultiIndexOnCollectionsWithNull() throws Throwable
    {
        createTable(" CREATE TABLE %s ( k int, v int, x text, l list<int>, s set<text>, m map<text, int>, PRIMARY KEY (k, v))");

        createIndex("CREATE INDEX ON %s (x)");
        createIndex("CREATE INDEX ON %s (v)");
        createIndex("CREATE INDEX ON %s (s)");
        createIndex("CREATE INDEX ON %s (m)");


        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (0, 0, 'x', [1, 2],    {'a'},      {'a' : 1})");
        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (0, 1, 'x', [3, 4],    {'b', 'c'}, {'a' : 1, 'b' : 2})");
        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (0, 2, 'x', [1],       {'a', 'c'}, {'c' : 3})");
        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (1, 0, 'x', [1, 2, 4], {},         {'b' : 1})");
        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (1, 1, 'x', [4, 5],    {'d'},      {'a' : 1, 'b' : 3})");
        execute("INSERT INTO %s (k, v, x, l, s, m) VALUES (1, 2, 'x', null,      null,       null)");

        beforeAndAfterFlush(() -> {
            // lists
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND l CONTAINS 1 ALLOW FILTERING"), row(1, 0), row(0, 0), row(0, 2));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND k = 0 AND l CONTAINS 1 ALLOW FILTERING"), row(0, 0), row(0, 2));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND l CONTAINS 2 ALLOW FILTERING"), row(1, 0), row(0, 0));
            assertEmpty(execute("SELECT k, v FROM %s WHERE x = 'x' AND l CONTAINS 6 ALLOW FILTERING"));

            // sets
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND s CONTAINS 'a' ALLOW FILTERING" ), row(0, 0), row(0, 2));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND k = 0 AND s CONTAINS 'a' ALLOW FILTERING"), row(0, 0), row(0, 2));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND s CONTAINS 'd' ALLOW FILTERING"), row(1, 1));
            assertEmpty(execute("SELECT k, v FROM %s  WHERE x = 'x' AND s CONTAINS 'e' ALLOW FILTERING"));

            // maps
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND m CONTAINS 1 ALLOW FILTERING"), row(1, 0), row(1, 1), row(0, 0), row(0, 1));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND k = 0 AND m CONTAINS 1 ALLOW FILTERING"), row(0, 0), row(0, 1));
            assertRows(execute("SELECT k, v FROM %s WHERE x = 'x' AND m CONTAINS 2 ALLOW FILTERING"), row(0, 1));
            assertEmpty(execute("SELECT k, v FROM %s  WHERE x = 'x' AND m CONTAINS 4 ALLOW FILTERING"));
        });
    }

    /**
     * Migrated from cql_tests.py:TestCQL.map_keys_indexing()
     */
    @Test
    public void testIndexOnMapKeys() throws Throwable
    {
        createTable("CREATE TABLE %s (k int, v int, m map<text, int>, PRIMARY KEY (k, v))");

        createIndex("CREATE INDEX ON %s(keys(m))");

        execute("INSERT INTO %s (k, v, m) VALUES (0, 0, {'a' : 1})");
        execute("INSERT INTO %s (k, v, m) VALUES (0, 1, {'a' : 1, 'b' : 2})");
        execute("INSERT INTO %s (k, v, m) VALUES (0, 2, {'c' : 3})");
        execute("INSERT INTO %s (k, v, m) VALUES (1, 0, {'b' : 1})");
        execute("INSERT INTO %s (k, v, m) VALUES (1, 1, {'a' : 1, 'b' : 3})");

        // maps
        assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS KEY 'a'"), row(1, 1), row(0, 0), row(0, 1));
        assertRows(execute("SELECT k, v FROM %s WHERE k = 0 AND m CONTAINS KEY 'a'"), row(0, 0), row(0, 1));
        assertRows(execute("SELECT k, v FROM %s WHERE m CONTAINS KEY 'c'"), row(0, 2));
        assertEmpty(execute("SELECT k, v FROM %s  WHERE m CONTAINS KEY 'd'"));
    }

    /**
     * Test for #6950 bug,
     * migrated from cql_tests.py:TestCQL.key_index_with_reverse_clustering()
     */
    @Test
    public void testIndexOnKeyWithReverseClustering() throws Throwable
    {
        createTable("CREATE TABLE %s (k1 int, k2 int, v int, PRIMARY KEY ((k1, k2), v) ) WITH CLUSTERING ORDER BY (v DESC)");

        createIndex("CREATE INDEX ON %s (k2)");

        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 0, 1)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 1, 2)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (0, 0, 3)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (1, 0, 4)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (1, 1, 5)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (2, 0, 7)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (2, 1, 8)");
        execute("INSERT INTO %s (k1, k2, v) VALUES (3, 0, 1)");

        assertRows(execute("SELECT * FROM %s WHERE k2 = 0 AND v >= 2 ALLOW FILTERING"),
                   row(2, 0, 7),
                   row(0, 0, 3),
                   row(1, 0, 4));
    }

    /**
     * Test for CASSANDRA-6612,
     * migrated from cql_tests.py:TestCQL.bug_6612_test()
     */
    @Test
    public void testSelectCountOnIndexedColumn() throws Throwable
    {
        createTable("CREATE TABLE %s (username text, session_id text, app_name text, account text, last_access timestamp, created_on timestamp, PRIMARY KEY (username, session_id, app_name, account))");

        createIndex("create index ON %s (app_name)");
        createIndex("create index ON %s (last_access)");

        assertRows(execute("select count(*) from %s where app_name='foo' and account='bar' and last_access > 4 allow filtering"), row(0L));

        execute("insert into %s (username, session_id, app_name, account, last_access, created_on) values ('toto', 'foo', 'foo', 'bar', 12, 13)");

        assertRows(execute("select count(*) from %s where app_name='foo' and account='bar' and last_access > 4 allow filtering"), row(1L));
    }

    @Test
    public void testSyntaxVariationsForIndexOnCollectionsValue() throws Throwable
    {
        createTable("CREATE TABLE %s (k int, m map<int, int>, l list<int>, s set<int>, PRIMARY KEY (k))");
        createAndDropCollectionValuesIndex("m");
        createAndDropCollectionValuesIndex("l");
        createAndDropCollectionValuesIndex("s");
    }

    private void createAndDropCollectionValuesIndex(String columnName) throws Throwable
    {
        String indexName = columnName + "_idx";
        SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
        createIndex(String.format("CREATE INDEX %s on %%s(%s)", indexName, columnName));
        IndexMetadata indexDef = indexManager.getIndexByName(indexName).getIndexMetadata();
        assertEquals(String.format("values(%s)", columnName), indexDef.options.get(IndexTarget.TARGET_OPTION_NAME));
        dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
        assertFalse(indexManager.hasIndexes());
        createIndex(String.format("CREATE INDEX %s on %%s(values(%s))", indexName, columnName));
        assertEquals(indexDef, indexManager.getIndexByName(indexName).getIndexMetadata());
        dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
    }

    @Test
    public void testCreateIndexWithQuotedColumnNames() throws Throwable
    {
        createTable("CREATE TABLE %s (" +
                    " k int," +
                    " v int, " +
                    " lower_case_map map<int, int>," +
                    " \"MixedCaseMap\" map<int, int>," +
                    " lower_case_frozen_list frozen<list<int>>," +
                    " \"UPPER_CASE_FROZEN_LIST\" frozen<list<int>>," +
                    " \"set name with spaces\" set<int>," +
                    " \"column_name_with\"\"escaped quote\" int," +
                    " PRIMARY KEY (k))");

        createAndDropIndexWithQuotedColumnIdentifier("\"v\"");
        createAndDropIndexWithQuotedColumnIdentifier("keys(\"lower_case_map\")");
        createAndDropIndexWithQuotedColumnIdentifier("keys(\"MixedCaseMap\")");
        createAndDropIndexWithQuotedColumnIdentifier("full(\"lower_case_frozen_list\")");
        createAndDropIndexWithQuotedColumnIdentifier("full(\"UPPER_CASE_FROZEN_LIST\")");
        createAndDropIndexWithQuotedColumnIdentifier("values(\"set name with spaces\")");
        createAndDropIndexWithQuotedColumnIdentifier("\"column_name_with\"\"escaped quote\"");
    }

    private void createAndDropIndexWithQuotedColumnIdentifier(String target) throws Throwable
    {
        String indexName = "test_mixed_case_idx";
        createIndex(String.format("CREATE INDEX %s ON %%s(%s)", indexName, target));
        SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
        IndexMetadata indexDef = indexManager.getIndexByName(indexName).getIndexMetadata();
        dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
        // verify we can re-create the index using the target string
        createIndex(String.format("CREATE INDEX %s ON %%s(%s)",
                                  indexName, indexDef.options.get(IndexTarget.TARGET_OPTION_NAME)));
        assertEquals(indexDef, indexManager.getIndexByName(indexName).getIndexMetadata());
        dropIndex(String.format("DROP INDEX %s.%s", KEYSPACE, indexName));
    }


    /**
     * Test for CASSANDRA-5732, Can not query secondary index
     * migrated from cql_tests.py:TestCQL.bug_5732_test(),
     * which was executing with a row cache size of 100 MB
     * and restarting the node, here we just cleanup the cache.
     */
    @Test
    public void testCanQuerySecondaryIndex() throws Throwable
    {
        String tableName = createTable("CREATE TABLE %s (k int PRIMARY KEY, v int,)");

        execute("ALTER TABLE %s WITH CACHING = { 'keys': 'ALL', 'rows_per_partition': 'ALL' }");
        execute("INSERT INTO %s (k,v) VALUES (0,0)");
        execute("INSERT INTO %s (k,v) VALUES (1,1)");

        createIndex("CREATE INDEX testindex on %s (v)");
        assertTrue(waitForIndex(keyspace(), tableName, "testindex"));

        assertRows(execute("SELECT k FROM %s WHERE v = 0"), row(0));
        cleanupCache();
        assertRows(execute("SELECT k FROM %s WHERE v = 0"), row(0));
    }

    // CASSANDRA-8280/8081
    // reject updates with indexed values where value > 64k
    // make sure we check conditional and unconditional statements,
    // both singly and in batches (CASSANDRA-10536)
    @Test
    public void testIndexOnCompositeValueOver64k() throws Throwable
    {
        createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a))");
        createIndex("CREATE INDEX ON %s(c)");
        failInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
        failInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?) IF NOT EXISTS", ByteBuffer.allocate(TOO_BIG));
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b, c) VALUES (0, 0, ?);\n" +
                   "APPLY BATCH",
                   ByteBuffer.allocate(TOO_BIG));
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b, c) VALUES (0, 0, ?) IF NOT EXISTS;\n" +
                   "APPLY BATCH",
                   ByteBuffer.allocate(TOO_BIG));
    }

    @Test
    public void testCompactTableWithValueOver64k() throws Throwable
    {
        createTable("CREATE TABLE %s(a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE");
        createIndex("CREATE INDEX ON %s(b)");
        failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", ByteBuffer.allocate(TOO_BIG));
        failInsert("INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS", ByteBuffer.allocate(TOO_BIG));
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b) VALUES (0, ?);\n" +
                   "APPLY BATCH",
                   ByteBuffer.allocate(TOO_BIG));
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS;\n" +
                   "APPLY BATCH",
                   ByteBuffer.allocate(TOO_BIG));
    }

    @Test
    public void testIndexOnPartitionKeyInsertValueOver64k() throws Throwable
    {
        createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY ((a, b)))");
        createIndex("CREATE INDEX ON %s(a)");
        succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?) IF NOT EXISTS", ByteBuffer.allocate(TOO_BIG));
        succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
        succeedInsert("BEGIN BATCH\n" +
                      "INSERT INTO %s (a, b, c) VALUES (0, 0, ?);\n" +
                      "APPLY BATCH", ByteBuffer.allocate(TOO_BIG));

        // the indexed value passes validation, but the batch size will
        // exceed the default failure threshold, so temporarily raise it
        // (the non-conditional batch doesn't hit this because
        // BatchStatement::executeInternal skips the size check but CAS
        // path does not)
        long batchSizeThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
        try
        {
            DatabaseDescriptor.setBatchSizeFailThresholdInKB( (TOO_BIG / 1024) * 2);
            succeedInsert("BEGIN BATCH\n" +
                          "INSERT INTO %s (a, b, c) VALUES (1, 1, ?) IF NOT EXISTS;\n" +
                          "APPLY BATCH", ByteBuffer.allocate(TOO_BIG));
        }
        finally
        {
            DatabaseDescriptor.setBatchSizeFailThresholdInKB((int) (batchSizeThreshold / 1024));
        }
    }

    @Test
    public void testIndexOnClusteringColumnInsertValueOver64k() throws Throwable
    {
        createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a, b))");
        createIndex("CREATE INDEX ON %s(b)");
        succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?) IF NOT EXISTS", ByteBuffer.allocate(TOO_BIG));
        succeedInsert("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)", ByteBuffer.allocate(TOO_BIG));
        succeedInsert("BEGIN BATCH\n" +
                      "INSERT INTO %s (a, b, c) VALUES (0, 0, ?);\n" +
                      "APPLY BATCH", ByteBuffer.allocate(TOO_BIG));

        // the indexed value passes validation, but the batch size will
        // exceed the default failure threshold, so temporarily raise it
        // (the non-conditional batch doesn't hit this because
        // BatchStatement::executeInternal skips the size check but CAS
        // path does not)
        long batchSizeThreshold = DatabaseDescriptor.getBatchSizeFailThreshold();
        try
        {
            DatabaseDescriptor.setBatchSizeFailThresholdInKB( (TOO_BIG / 1024) * 2);
            succeedInsert("BEGIN BATCH\n" +
                          "INSERT INTO %s (a, b, c) VALUES (1, 1, ?) IF NOT EXISTS;\n" +
                          "APPLY BATCH", ByteBuffer.allocate(TOO_BIG));
        }
        finally
        {
            DatabaseDescriptor.setBatchSizeFailThresholdInKB((int)(batchSizeThreshold / 1024));
        }
    }

    @Test
    public void testIndexOnFullCollectionEntryInsertCollectionValueOver64k() throws Throwable
    {
        createTable("CREATE TABLE %s(a int, b frozen<map<int, blob>>, PRIMARY KEY (a))");
        createIndex("CREATE INDEX ON %s(full(b))");
        Map<Integer, ByteBuffer> map = new HashMap<>();
        map.put(0, ByteBuffer.allocate(1024 * 65));
        failInsert("INSERT INTO %s (a, b) VALUES (0, ?)", map);
        failInsert("INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS", map);
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b) VALUES (0, ?);\n" +
                   "APPLY BATCH", map);
        failInsert("BEGIN BATCH\n" +
                   "INSERT INTO %s (a, b) VALUES (0, ?) IF NOT EXISTS;\n" +
                   "APPLY BATCH", map);
    }

    public void failInsert(String insertCQL, Object...args) throws Throwable
    {
        try
        {
            execute(insertCQL, args);
            fail("Expected statement to fail validation");
        }
        catch (Exception e)
        {
            // as expected
        }
    }

    public void succeedInsert(String insertCQL, Object...args) throws Throwable
    {
        execute(insertCQL, args);
        flush();
    }

    /**
     * Migrated from cql_tests.py:TestCQL.clustering_indexing_test()
     */
    @Test
    public void testIndexesOnClustering() throws Throwable
    {
        createTable("CREATE TABLE %s ( id1 int, id2 int, author text, time bigint, v1 text, v2 text, PRIMARY KEY ((id1, id2), author, time))");

        createIndex("CREATE INDEX ON %s (time)");
        execute("CREATE INDEX ON %s (id2)");

        execute("INSERT INTO %s (id1, id2, author, time, v1, v2) VALUES(0, 0, 'bob', 0, 'A', 'A')");
        execute("INSERT INTO %s (id1, id2, author, time, v1, v2) VALUES(0, 0, 'bob', 1, 'B', 'B')");
        execute("INSERT INTO %s (id1, id2, author, time, v1, v2) VALUES(0, 1, 'bob', 2, 'C', 'C')");
        execute("INSERT INTO %s (id1, id2, author, time, v1, v2) VALUES(0, 0, 'tom', 0, 'D', 'D')");
        execute("INSERT INTO %s (id1, id2, author, time, v1, v2) VALUES(0, 1, 'tom', 1, 'E', 'E')");

        assertRows(execute("SELECT v1 FROM %s WHERE time = 1"),
                   row("B"), row("E"));

        assertRows(execute("SELECT v1 FROM %s WHERE id2 = 1"),
                   row("C"), row("E"));

        assertRows(execute("SELECT v1 FROM %s WHERE id1 = 0 AND id2 = 0 AND author = 'bob' AND time = 0"),
                   row("A"));

        // Test for CASSANDRA-8206
        execute("UPDATE %s SET v2 = null WHERE id1 = 0 AND id2 = 0 AND author = 'bob' AND time = 1");

        assertRows(execute("SELECT v1 FROM %s WHERE id2 = 0"),
                   row("A"), row("B"), row("D"));

        assertRows(execute("SELECT v1 FROM %s WHERE time = 1"),
                   row("B"), row("E"));
    }

    /**
     * Migrated from cql_tests.py:TestCQL.invalid_clustering_indexing_test()
     */
    @Test
    public void testIndexesOnClusteringInvalid() throws Throwable
    {
        createTable("CREATE TABLE %s (a int, b int, c int, d int, PRIMARY KEY ((a, b))) WITH COMPACT STORAGE");
        assertInvalid("CREATE INDEX ON %s (a)");
        assertInvalid("CREATE INDEX ON %s (b)");

        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b)) WITH COMPACT STORAGE");
        assertInvalid("CREATE INDEX ON %s (a)");
        assertInvalid("CREATE INDEX ON %s (b)");
        assertInvalid("CREATE INDEX ON %s (c)");

        createTable("CREATE TABLE %s (a int, b int, c int static , PRIMARY KEY (a, b))");
        assertInvalid("CREATE INDEX ON %s (c)");
    }

    @Test
    public void testMultipleIndexesOnOneColumn() throws Throwable
    {
        String indexClassName = StubIndex.class.getName();
        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
        // uses different options otherwise the two indexes are considered duplicates
        createIndex(String.format("CREATE CUSTOM INDEX c_idx_1 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'a'}", indexClassName));
        createIndex(String.format("CREATE CUSTOM INDEX c_idx_2 ON %%s(c) USING '%s' WITH OPTIONS = {'foo':'b'}", indexClassName));

        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
        CFMetaData cfm = cfs.metadata;
        StubIndex index1 = (StubIndex)cfs.indexManager.getIndex(cfm.getIndexes()
                                                                   .get("c_idx_1")
                                                                   .orElseThrow(throwAssert("index not found")));
        StubIndex index2 = (StubIndex)cfs.indexManager.getIndex(cfm.getIndexes()
                                                                   .get("c_idx_2")
                                                                   .orElseThrow(throwAssert("index not found")));
        Object[] row1a = row(0, 0, 0);
        Object[] row1b = row(0, 0, 1);
        Object[] row2 = row(2, 2, 2);
        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", row1a);
        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", row1b);
        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?)", row2);

        assertEquals(2, index1.rowsInserted.size());
        assertColumnValue(0, "c", index1.rowsInserted.get(0), cfm);
        assertColumnValue(2, "c", index1.rowsInserted.get(1), cfm);

        assertEquals(2, index2.rowsInserted.size());
        assertColumnValue(0, "c", index2.rowsInserted.get(0), cfm);
        assertColumnValue(2, "c", index2.rowsInserted.get(1), cfm);

        assertEquals(1, index1.rowsUpdated.size());
        assertColumnValue(0, "c", index1.rowsUpdated.get(0).left, cfm);
        assertColumnValue(1, "c", index1.rowsUpdated.get(0).right, cfm);

        assertEquals(1, index2.rowsUpdated.size());
        assertColumnValue(0, "c", index2.rowsUpdated.get(0).left, cfm);
        assertColumnValue(1, "c", index2.rowsUpdated.get(0).right, cfm);
    }

    @Test
    public void testDeletions() throws Throwable
    {
        // Test for bugs like CASSANDRA-10694.  These may not be readily visible with the built-in secondary index
        // implementation because of the stale entry handling.

        String indexClassName = StubIndex.class.getName();
        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
        createIndex(String.format("CREATE CUSTOM INDEX c_idx ON %%s(c) USING '%s'", indexClassName));

        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
        CFMetaData cfm = cfs.metadata;
        StubIndex index1 = (StubIndex) cfs.indexManager.getIndex(cfm.getIndexes()
                .get("c_idx")
                .orElseThrow(throwAssert("index not found")));

        execute("INSERT INTO %s (a, b, c) VALUES (?, ?, ?) USING TIMESTAMP 1", 0, 0, 0);
        assertEquals(1, index1.rowsInserted.size());

        execute("DELETE FROM %s USING TIMESTAMP 2 WHERE a = ? AND b = ?", 0, 0);
        assertEquals(1, index1.rowsUpdated.size());
        Pair<Row, Row> update = index1.rowsUpdated.get(0);
        Row existingRow = update.left;
        Row newRow = update.right;

        // check the existing row from the update call
        assertTrue(existingRow.deletion().isLive());
        assertEquals(DeletionTime.LIVE, existingRow.deletion().time());
        assertEquals(1L, existingRow.primaryKeyLivenessInfo().timestamp());

        // check the new row from the update call
        assertFalse(newRow.deletion().isLive());
        assertEquals(2L, newRow.deletion().time().markedForDeleteAt());
        assertFalse(newRow.cells().iterator().hasNext());

        // delete the same row again
        execute("DELETE FROM %s USING TIMESTAMP 3 WHERE a = ? AND b = ?", 0, 0);
        assertEquals(2, index1.rowsUpdated.size());
        update = index1.rowsUpdated.get(1);
        existingRow = update.left;
        newRow = update.right;

        // check the new row from the update call
        assertFalse(existingRow.deletion().isLive());
        assertEquals(2L, existingRow.deletion().time().markedForDeleteAt());
        assertFalse(existingRow.cells().iterator().hasNext());

        // check the new row from the update call
        assertFalse(newRow.deletion().isLive());
        assertEquals(3L, newRow.deletion().time().markedForDeleteAt());
        assertFalse(newRow.cells().iterator().hasNext());
    }

    @Test
    public void testUpdatesToMemtableData() throws Throwable
    {
        // verify the contract specified by Index.Indexer::updateRow(oldRowData, newRowData),
        // when a row in the memtable is updated, the indexer should be informed of:
        // * new columns
        // * removed columns
        // * columns whose value, timestamp or ttl have been modified.
        // Any columns which are unchanged by the update are not passed to the Indexer
        // Note that for simplicity this test resets the index between each scenario
        createTable("CREATE TABLE %s (k int, c int, v1 int, v2 int, PRIMARY KEY (k,c))");
        createIndex(String.format("CREATE CUSTOM INDEX test_index ON %%s() USING '%s'", StubIndex.class.getName()));
        execute("INSERT INTO %s (k, c, v1, v2) VALUES (0, 0, 0, 0) USING TIMESTAMP 0");

        ColumnDefinition v1 = getCurrentColumnFamilyStore().metadata.getColumnDefinition(new ColumnIdentifier("v1", true));
        ColumnDefinition v2 = getCurrentColumnFamilyStore().metadata.getColumnDefinition(new ColumnIdentifier("v2", true));

        StubIndex index = (StubIndex)getCurrentColumnFamilyStore().indexManager.getIndexByName("test_index");
        assertEquals(1, index.rowsInserted.size());

        // Overwrite a single value, leaving the other untouched
        execute("UPDATE %s USING TIMESTAMP 1 SET v1=1 WHERE k=0 AND c=0");
        assertEquals(1, index.rowsUpdated.size());
        Row oldRow = index.rowsUpdated.get(0).left;
        assertEquals(1, oldRow.columnCount());
        validateCell(oldRow.getCell(v1), v1, ByteBufferUtil.bytes(0), 0);
        Row newRow = index.rowsUpdated.get(0).right;
        assertEquals(1, newRow.columnCount());
        validateCell(newRow.getCell(v1), v1, ByteBufferUtil.bytes(1), 1);
        index.reset();

        // Overwrite both values
        execute("UPDATE %s USING TIMESTAMP 2 SET v1=2, v2=2 WHERE k=0 AND c=0");
        assertEquals(1, index.rowsUpdated.size());
        oldRow = index.rowsUpdated.get(0).left;
        assertEquals(2, oldRow.columnCount());
        validateCell(oldRow.getCell(v1), v1, ByteBufferUtil.bytes(1), 1);
        validateCell(oldRow.getCell(v2), v2, ByteBufferUtil.bytes(0), 0);
        newRow = index.rowsUpdated.get(0).right;
        assertEquals(2, newRow.columnCount());
        validateCell(newRow.getCell(v1), v1, ByteBufferUtil.bytes(2), 2);
        validateCell(newRow.getCell(v2), v2, ByteBufferUtil.bytes(2), 2);
        index.reset();

        // Delete one value
        execute("DELETE v1 FROM %s USING TIMESTAMP 3 WHERE k=0 AND c=0");
        assertEquals(1, index.rowsUpdated.size());
        oldRow = index.rowsUpdated.get(0).left;
        assertEquals(1, oldRow.columnCount());
        validateCell(oldRow.getCell(v1), v1, ByteBufferUtil.bytes(2), 2);
        newRow = index.rowsUpdated.get(0).right;
        assertEquals(1, newRow.columnCount());
        Cell newCell = newRow.getCell(v1);
        assertTrue(newCell.isTombstone());
        assertEquals(3, newCell.timestamp());
        index.reset();

        // Modify the liveness of the primary key, the delta rows should contain
        // no cell data as only the pk was altered, but it should illustrate the
        // change to the liveness info
        execute("INSERT INTO %s(k, c) VALUES (0, 0) USING TIMESTAMP 4");
        assertEquals(1, index.rowsUpdated.size());
        oldRow = index.rowsUpdated.get(0).left;
        assertEquals(0, oldRow.columnCount());
        assertEquals(0, oldRow.primaryKeyLivenessInfo().timestamp());
        newRow = index.rowsUpdated.get(0).right;
        assertEquals(0, newRow.columnCount());
        assertEquals(4, newRow.primaryKeyLivenessInfo().timestamp());
    }

    @Test
    public void testIndexQueriesWithIndexNotReady() throws Throwable
    {
        createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");

        for (int i = 0; i < 10; i++)
            for (int j = 0; j < 10; j++)
                execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", i, j, i + j);

        createIndex("CREATE CUSTOM INDEX testIndex ON %s (value) USING '" + IndexBlockingOnInitialization.class.getName() + "'");
        try
        {
            execute("SELECT value FROM %s WHERE value = 2");
            fail();
        }
        catch (IndexNotAvailableException e)
        {
            assertTrue(true);
        }
        finally
        {
            execute("DROP index " + KEYSPACE + ".testIndex");
        }
    }

    @Test
    public void droppingIndexInvalidatesPreparedStatements() throws Throwable
    {
        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY ((a), b))");
        createIndex("CREATE INDEX c_idx ON %s(c)");
        MD5Digest cqlId = prepareStatement("SELECT * FROM %s.%s WHERE c=?", false).statementId;
        Integer thriftId = prepareStatement("SELECT * FROM %s.%s WHERE c=?", true).toThriftPreparedResult().getItemId();

        assertNotNull(QueryProcessor.instance.getPrepared(cqlId));
        assertNotNull(QueryProcessor.instance.getPreparedForThrift(thriftId));

        dropIndex("DROP INDEX %s.c_idx");

        assertNull(QueryProcessor.instance.getPrepared(cqlId));
        assertNull(QueryProcessor.instance.getPreparedForThrift(thriftId));
    }

    // See CASSANDRA-11021
    @Test
    public void testIndexesOnNonStaticColumnsWhereSchemaIncludesStaticColumns() throws Throwable
    {
        createTable("CREATE TABLE %s (a int, b int, c int static, d int, PRIMARY KEY (a, b))");
        createIndex("CREATE INDEX b_idx on %s(b)");
        createIndex("CREATE INDEX d_idx on %s(d)");

        execute("INSERT INTO %s (a, b, c ,d) VALUES (0, 0, 0, 0)");
        execute("INSERT INTO %s (a, b, c, d) VALUES (1, 1, 1, 1)");
        assertRows(execute("SELECT * FROM %s WHERE b = 0"), row(0, 0, 0, 0));
        assertRows(execute("SELECT * FROM %s WHERE d = 1"), row(1, 1, 1, 1));

        execute("UPDATE %s SET c = 2 WHERE a = 0");
        execute("UPDATE %s SET c = 3, d = 4 WHERE a = 1 AND b = 1");
        assertRows(execute("SELECT * FROM %s WHERE b = 0"), row(0, 0, 2, 0));
        assertRows(execute("SELECT * FROM %s WHERE d = 4"), row(1, 1, 3, 4));

        execute("DELETE FROM %s WHERE a = 0");
        execute("DELETE FROM %s WHERE a = 1 AND b = 1");
        assertEmpty(execute("SELECT * FROM %s WHERE b = 0"));
        assertEmpty(execute("SELECT * FROM %s WHERE d = 3"));
    }

    @Test
    public void testWithEmptyRestrictionValueAndSecondaryIndex() throws Throwable
    {
        createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c))");
        createIndex("CREATE INDEX on %s(c)");
        createIndex("CREATE INDEX on %s(v)");

        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("1"), bytes("1"));
        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)", bytes("foo123"), bytes("2"), bytes("1"));

        beforeAndAfterFlush(() -> {
            // Test clustering columns restrictions
            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
                       row(bytes("foo123"), bytes("1"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
                       row(bytes("foo123"), bytes("1"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) > (textAsBlob('')) AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));
        });

        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
                bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1"));

        beforeAndAfterFlush(() -> {

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c = textAsBlob('');"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) = (textAsBlob(''));"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c IN (textAsBlob(''), textAsBlob('1'));"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
                       row(bytes("foo123"), bytes("1"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) IN ((textAsBlob('')), (textAsBlob('1')));"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
                       row(bytes("foo123"), bytes("1"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c > textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")),
                       row(bytes("foo123"), bytes("1"), bytes("1")),
                       row(bytes("foo123"), bytes("2"), bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c <= textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) <= (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"),
                       row(bytes("foo123"), EMPTY_BYTE_BUFFER, bytes("1")));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) < (textAsBlob('')) AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND c >= textAsBlob('') AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND (c) >= (textAsBlob('')) AND c < textAsBlob('') AND v = textAsBlob('1') ALLOW FILTERING;"));

            // Test restrictions on non-primary key value
            assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"));
        });

        execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?)",
                bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER);

        beforeAndAfterFlush(() -> {

            assertRows(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"),
                       row(bytes("foo123"), bytes("3"), EMPTY_BYTE_BUFFER));
        });
    }

    @Test
    public void testEmptyRestrictionValueWithSecondaryIndexAndCompactTables() throws Throwable
    {
        createTable("CREATE TABLE %s (pk blob, c blob, v blob, PRIMARY KEY ((pk), c)) WITH COMPACT STORAGE");
        assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
                            "CREATE INDEX on %s(c)");

        createTable("CREATE TABLE %s (pk blob PRIMARY KEY, v blob) WITH COMPACT STORAGE");
        createIndex("CREATE INDEX on %s(v)");

        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo123"), bytes("1"));

        // Test restrictions on non-primary key value
        assertEmpty(execute("SELECT * FROM %s WHERE pk = textAsBlob('foo123') AND v = textAsBlob('');"));

        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", bytes("foo124"), EMPTY_BYTE_BUFFER);

        assertRows(execute("SELECT * FROM %s WHERE v = textAsBlob('');"),
                   row(bytes("foo124"), EMPTY_BYTE_BUFFER));
    }

    @Test
    public void testIndexOnPartitionKeyWithStaticColumnAndNoRows() throws Throwable
    {
        createTable("CREATE TABLE %s (pk1 int, pk2 int, c int, s int static, v int, PRIMARY KEY((pk1, pk2), c))");
        createIndex("CREATE INDEX ON %s (pk2)");
        execute("INSERT INTO %s (pk1, pk2, c, s, v) VALUES (?, ?, ?, ?, ?)", 1, 1, 1, 9, 1);
        execute("INSERT INTO %s (pk1, pk2, c, s, v) VALUES (?, ?, ?, ?, ?)", 1, 1, 2, 9, 2);
        execute("INSERT INTO %s (pk1, pk2, s) VALUES (?, ?, ?)", 2, 1, 9);
        execute("INSERT INTO %s (pk1, pk2, c, s, v) VALUES (?, ?, ?, ?, ?)", 3, 1, 1, 9, 1);

        assertRows(execute("SELECT * FROM %s WHERE pk2 = ?", 1),
                   row(2, 1, null, 9, null),
                   row(1, 1, 1, 9, 1),
                   row(1, 1, 2, 9, 2),
                   row(3, 1, 1, 9, 1));

        execute("UPDATE %s SET s=?, v=? WHERE pk1=? AND pk2=? AND c=?", 9, 1, 1, 10, 2);
        assertRows(execute("SELECT * FROM %s WHERE pk2 = ?", 10), row(1, 10, 2, 9, 1));

        execute("UPDATE %s SET s=? WHERE pk1=? AND pk2=?", 9, 1, 20);
        assertRows(execute("SELECT * FROM %s WHERE pk2 = ?", 20), row(1, 20, null, 9, null));
    }

    private ResultMessage.Prepared prepareStatement(String cql, boolean forThrift)
    {
        return QueryProcessor.prepare(String.format(cql, KEYSPACE, currentTable()),
                                      ClientState.forInternalCalls(),
                                      forThrift);
    }

    private void validateCell(Cell cell, ColumnDefinition def, ByteBuffer val, long timestamp)
    {
        assertNotNull(cell);
        assertEquals(0, def.type.compare(cell.value(), val));
        assertEquals(timestamp, cell.timestamp());
    }

    private static void assertColumnValue(int expected, String name, Row row, CFMetaData cfm)
    {
        ColumnDefinition col = cfm.getColumnDefinition(new ColumnIdentifier(name, true));
        AbstractType<?> type = col.type;
        assertEquals(expected, type.compose(row.getCell(col).value()));
    }

    /**
     * <code>CassandraIndex</code> that blocks during the initialization.
     */
    public static class IndexBlockingOnInitialization extends CustomCassandraIndex
    {
        private final CountDownLatch latch = new CountDownLatch(1);

        public IndexBlockingOnInitialization(ColumnFamilyStore baseCfs, IndexMetadata indexDef)
        {
            super(baseCfs, indexDef);
        }

        @Override
        public Callable<?> getInitializationTask()
        {
            return () -> {
                latch.await();
                return null;
            };
        }

        @Override
        public Callable<?> getInvalidateTask()
        {
            latch.countDown();
            return super.getInvalidateTask();
        }
    }

    @Test
    public void testIndexOnPartitionKeyInsertExpiringColumn() throws Throwable
    {
        createTable("CREATE TABLE %s (k1 int, k2 int, a int, b int, PRIMARY KEY ((k1, k2)))");
        createIndex("CREATE INDEX on %s(k1)");
        execute("INSERT INTO %s (k1, k2, a, b) VALUES (1, 2, 3, 4)");
        assertRows(execute("SELECT * FROM %s WHERE k1 = 1"), row(1, 2, 3, 4));
        execute("UPDATE %s USING TTL 1 SET b = 10 WHERE k1 = 1 AND k2 = 2");
        Thread.sleep(1000);
        assertRows(execute("SELECT * FROM %s WHERE k1 = 1"), row(1, 2, 3, null));
    }

    @Test
    public void testIndexOnClusteringKeyInsertExpiringColumn() throws Throwable
    {
        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
        createIndex("CREATE INDEX on %s(ck)");
        execute("INSERT INTO %s (pk, ck, a, b) VALUES (1, 2, 3, 4)");
        assertRows(execute("SELECT * FROM %s WHERE ck = 2"), row(1, 2, 3, 4));
        execute("UPDATE %s USING TTL 1 SET b = 10 WHERE pk = 1 AND ck = 2");
        Thread.sleep(1000);
        assertRows(execute("SELECT * FROM %s WHERE ck = 2"), row(1, 2, 3, null));
    }

    @Test
    public void testIndexOnRegularColumnInsertExpiringColumn() throws Throwable
    {
        createTable("CREATE TABLE %s (pk int, ck int, a int, b int, PRIMARY KEY (pk, ck))");
        createIndex("CREATE INDEX on %s(a)");
        execute("INSERT INTO %s (pk, ck, a, b) VALUES (1, 2, 3, 4)");
        assertRows(execute("SELECT * FROM %s WHERE a = 3"), row(1, 2, 3, 4));

        execute("UPDATE %s USING TTL 1 SET b = 10 WHERE pk = 1 AND ck = 2");
        Thread.sleep(1000);
        assertRows(execute("SELECT * FROM %s WHERE a = 3"), row(1, 2, 3, null));

        execute("UPDATE %s USING TTL 1 SET a = 5 WHERE pk = 1 AND ck = 2");
        Thread.sleep(1000);
        assertEmpty(execute("SELECT * FROM %s WHERE a = 3"));
        assertEmpty(execute("SELECT * FROM %s WHERE a = 5"));
    }

    @Test
    public void testIndicesOnCompactTable() throws Throwable
    {
        assertInvalidMessage("COMPACT STORAGE with composite PRIMARY KEY allows no more than one column not part of the PRIMARY KEY (got: v1, v2)",
                             "CREATE TABLE test (pk int, c int, v1 int, v2 int, PRIMARY KEY(pk, c)) WITH COMPACT STORAGE");

        createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c)) WITH COMPACT STORAGE");
        assertInvalidMessage("Secondary indexes are not supported on COMPACT STORAGE tables that have clustering columns",
                             "CREATE INDEX ON %s(v)");

        createTable("CREATE TABLE %s (pk int PRIMARY KEY, v int) WITH COMPACT STORAGE");
        createIndex("CREATE INDEX ON %s(v)");

        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 1, 1);
        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 2, 1);
        execute("INSERT INTO %s (pk, v) VALUES (?, ?)", 3, 3);

        assertRows(execute("SELECT pk, v FROM %s WHERE v = 1"),
                   row(1, 1),
                   row(2, 1));

        assertRows(execute("SELECT pk, v FROM %s WHERE v = 3"),
                   row(3, 3));

        assertEmpty(execute("SELECT pk, v FROM %s WHERE v = 5"));

        createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int) WITH COMPACT STORAGE");
        createIndex("CREATE INDEX ON %s(v1)");

        execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 1, 1, 1);
        execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 2, 1, 2);
        execute("INSERT INTO %s (pk, v1, v2) VALUES (?, ?, ?)", 3, 3, 3);

        assertRows(execute("SELECT pk, v2 FROM %s WHERE v1 = 1"),
                   row(1, 1),
                   row(2, 2));

        assertRows(execute("SELECT pk, v2 FROM %s WHERE v1 = 3"),
                   row(3, 3));

        assertEmpty(execute("SELECT pk, v2 FROM %s WHERE v1 = 5"));
    }
}