blob: bfa9ed6e38def50d49b885c22f7c84339e5aea75 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.end2end.index.IndexCoprocIT.INDEXER_CONFIG;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.hbase.index.IndexRegionObserver;
import org.apache.phoenix.hbase.index.Indexer;
import org.apache.phoenix.hbase.index.covered.NonTxIndexBuilder;
import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
import org.apache.phoenix.index.GlobalIndexChecker;
import org.apache.phoenix.index.PhoenixIndexBuilder;
import org.apache.phoenix.index.PhoenixIndexCodec;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PRow;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.util.EncodedColumnsUtil;
import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.junit.Assert;
public class IndexTestUtil {
// the normal db metadata interface is insufficient for all fields needed for an
// index table test.
private static final String SELECT_DATA_INDEX_ROW = "SELECT " + COLUMN_FAMILY
+ " FROM "
+ "\""+SYSTEM_CATALOG_SCHEMA +"\""+ ".\"" + SYSTEM_CATALOG_TABLE
+ "\" WHERE "
+ TENANT_ID + " IS NULL AND " + TABLE_SCHEM + "=? AND " + TABLE_NAME + "=? AND " + COLUMN_NAME + " IS NULL AND " + COLUMN_FAMILY + "=?";
public static ResultSet readDataTableIndexRow(Connection conn, String schemaName, String tableName, String indexName) throws SQLException {
PreparedStatement stmt = conn.prepareStatement(SELECT_DATA_INDEX_ROW);
stmt.setString(1, schemaName);
stmt.setString(2, tableName);
stmt.setString(3, indexName);
return stmt.executeQuery();
}
private static void coerceDataValueToIndexValue(PColumn dataColumn, PColumn indexColumn, ImmutableBytesWritable ptr) {
PDataType dataType = dataColumn.getDataType();
// TODO: push to RowKeySchema?
SortOrder dataModifier = dataColumn.getSortOrder();
PDataType indexType = indexColumn.getDataType();
SortOrder indexModifier = indexColumn.getSortOrder();
// We know ordinal position will match pk position, because you cannot
// alter an index table.
indexType.coerceBytes(ptr, dataType, dataModifier, indexModifier);
}
public static List<Mutation> generateIndexData(PTable index, PTable table,
List<Mutation> dataMutations, ImmutableBytesWritable ptr, KeyValueBuilder builder)
throws SQLException {
List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
for (Mutation dataMutation : dataMutations) {
indexMutations.addAll(generateIndexData(index, table, dataMutation, ptr, builder));
}
return indexMutations;
}
public static List<Mutation> generateIndexData(PTable indexTable, PTable dataTable,
Mutation dataMutation, ImmutableBytesWritable ptr, KeyValueBuilder builder)
throws SQLException {
byte[] dataRowKey = dataMutation.getRow();
RowKeySchema dataRowKeySchema = dataTable.getRowKeySchema();
List<PColumn> dataPKColumns = dataTable.getPKColumns();
int i = 0;
int indexOffset = 0;
Boolean hasValue;
// Skip salt column
int maxOffset = dataRowKey.length;
dataRowKeySchema.iterator(dataRowKey, ptr, dataTable.getBucketNum() == null ? i : ++i);
List<PColumn> indexPKColumns = indexTable.getPKColumns();
List<PColumn> indexColumns = indexTable.getColumns();
int nIndexColumns = indexPKColumns.size();
int maxIndexValues = indexColumns.size() - nIndexColumns - indexOffset;
BitSet indexValuesSet = new BitSet(maxIndexValues);
byte[][] indexValues = new byte[indexColumns.size() - indexOffset][];
while ((hasValue = dataRowKeySchema.next(ptr, i, maxOffset)) != null) {
if (hasValue) {
PColumn dataColumn = dataPKColumns.get(i);
PColumn indexColumn = indexTable.getColumnForColumnName(IndexUtil.getIndexColumnName(dataColumn));
coerceDataValueToIndexValue(dataColumn, indexColumn, ptr);
indexValues[indexColumn.getPosition()-indexOffset] = ptr.copyBytes();
}
i++;
}
PRow row;
long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
if (dataMutation instanceof Delete && dataMutation.getFamilyCellMap().values().isEmpty()) {
indexTable.newKey(ptr, indexValues);
row = indexTable.newRow(builder, ts, ptr, false);
row.delete();
} else {
// If no column families in table, then nothing to look for
if (!dataTable.getColumnFamilies().isEmpty()) {
for (Map.Entry<byte[],List<Cell>> entry : dataMutation.getFamilyCellMap().entrySet()) {
PColumnFamily family = dataTable.getColumnFamily(entry.getKey());
for (Cell kv : entry.getValue()) {
byte[] cq = CellUtil.cloneQualifier(kv);
byte[] emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(dataTable).getFirst();
if (Bytes.compareTo(emptyKVQualifier, cq) != 0) {
try {
PColumn dataColumn = family.getPColumnForColumnQualifier(cq);
PColumn indexColumn = indexTable.getColumnForColumnName(IndexUtil.getIndexColumnName(family.getName().getString(), dataColumn.getName().getString()));
ptr.set(kv.getValueArray(),kv.getValueOffset(),kv.getValueLength());
coerceDataValueToIndexValue(dataColumn, indexColumn, ptr);
indexValues[indexPKColumns.indexOf(indexColumn)-indexOffset] = ptr.copyBytes();
if (!SchemaUtil.isPKColumn(indexColumn)) {
indexValuesSet.set(indexColumn.getPosition()-nIndexColumns-indexOffset);
}
} catch (ColumnNotFoundException e) {
// Ignore as this means that the data column isn't in the index
}
}
}
}
}
indexTable.newKey(ptr, indexValues);
row = indexTable.newRow(builder, ts, ptr, false);
int pos = 0;
while ((pos = indexValuesSet.nextSetBit(pos)) >= 0) {
int index = nIndexColumns + indexOffset + pos++;
PColumn indexColumn = indexColumns.get(index);
row.setValue(indexColumn, indexValues[index]);
}
}
return row.toRowMutations();
}
public static void downgradeCoprocs(String physicalTableName,
String physicalIndexName, Admin admin) throws Exception {
TableDescriptor baseDescriptor = admin.getDescriptor(TableName.valueOf(physicalTableName));
TableDescriptorBuilder baseDescBuilder = TableDescriptorBuilder.newBuilder(baseDescriptor);
assertCoprocsContains(IndexRegionObserver.class, baseDescriptor);
removeCoproc(IndexRegionObserver.class, baseDescBuilder, admin);
if (physicalIndexName != null) {
TableDescriptor indexDescriptor =
admin.getDescriptor(TableName.valueOf(physicalIndexName));
assertCoprocsContains(GlobalIndexChecker.class, indexDescriptor);
TableDescriptorBuilder indexDescBuilder =
TableDescriptorBuilder.newBuilder(indexDescriptor);
removeCoproc(IndexRegionObserver.class, indexDescBuilder, admin);
removeCoproc(GlobalIndexChecker.class, indexDescBuilder, admin);
}
Map<String, String> props = new HashMap<>();
props.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, PhoenixIndexCodec.class.getName());
Indexer.enableIndexing(baseDescBuilder, PhoenixIndexBuilder.class,
props, QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
admin.modifyTable(baseDescBuilder.build());
baseDescriptor = admin.getDescriptor(TableName.valueOf(physicalTableName));
TableDescriptor indexDescriptor = null;
if (physicalIndexName != null) {
indexDescriptor = admin.getDescriptor(TableName.valueOf(physicalIndexName));
}
assertUsingOldCoprocs(baseDescriptor, indexDescriptor);
}
public static void assertCoprocsContains(Class clazz, TableDescriptor descriptor) {
String expectedCoprocName = clazz.getName();
boolean foundCoproc = isCoprocPresent(descriptor, expectedCoprocName);
Assert.assertTrue("Could not find coproc " + expectedCoprocName +
" in descriptor " + descriptor,foundCoproc);
}
public static boolean isCoprocPresent(TableDescriptor descriptor, String expectedCoprocName) {
boolean foundCoproc = false;
for (String coprocName : descriptor.getCoprocessors()){
if (coprocName.equals(expectedCoprocName)){
foundCoproc = true;
break;
}
}
return foundCoproc;
}
public static void removeCoproc(Class clazz, TableDescriptorBuilder descBuilder, Admin admin)
throws Exception {
descBuilder.removeCoprocessor(clazz.getName());
admin.modifyTable(descBuilder.build());
}
public static void assertUsingOldCoprocs(TableDescriptor baseDescriptor,
TableDescriptor indexDescriptor) {
assertCoprocsContains(Indexer.class, baseDescriptor);
assertCoprocConfig(baseDescriptor, Indexer.class.getName(),
INDEXER_CONFIG);
assertCoprocsNotContains(IndexRegionObserver.class, baseDescriptor);
if (indexDescriptor != null) {
assertCoprocsNotContains(IndexRegionObserver.class, indexDescriptor);
assertCoprocsNotContains(GlobalIndexChecker.class, indexDescriptor);
}
}
public static void assertCoprocConfig(TableDescriptor indexDesc,
String className, String expectedConfigValue){
boolean foundConfig = false;
for (Map.Entry<Bytes, Bytes> entry :
indexDesc.getValues().entrySet()){
String propKey = Bytes.toString(entry.getKey().get());
String propValue = Bytes.toString(entry.getValue().get());
//Unfortunately, a good API to read coproc properties didn't show up until
//HBase 2.0. Doing this the painful String-matching way to be compatible with 1.x
if (propKey.contains("coprocessor")){
if (propValue.contains(className)){
Assert.assertEquals(className + " is configured incorrectly",
expectedConfigValue,
propValue);
foundConfig = true;
break;
}
}
}
Assert.assertTrue("Couldn't find config for " + className, foundConfig);
}
public static void assertCoprocsNotContains(Class clazz, TableDescriptor descriptor) {
String expectedCoprocName = clazz.getName();
boolean foundCoproc = isCoprocPresent(descriptor, expectedCoprocName);
Assert.assertFalse("Could find coproc " + expectedCoprocName +
" in descriptor " + descriptor,foundCoproc);
}
}