PHOENIX-4332 Indexes should inherit guide post width of the base data table
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java
index d5d8442..eb01e89 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableNonTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class ColumnEncodedImmutableNonTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java
index 23b1654..4e90d70 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedImmutableTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class ColumnEncodedImmutableTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableNonTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableNonTxStatsCollectorIT.java
index 24869a2..2a560db 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableNonTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableNonTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class ColumnEncodedMutableNonTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java
index eea591d..01fa2b5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ColumnEncodedMutableTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class ColumnEncodedMutableTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableNonTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableNonTxStatsCollectorIT.java
index fe70030..27c6dc2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableNonTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableNonTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class NonColumnEncodedImmutableNonTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java
index 10a846a..0cec31a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonColumnEncodedImmutableTxStatsCollectorIT.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.junit.runners.Parameterized.Parameters;
public class NonColumnEncodedImmutableTxStatsCollectorIT extends StatsCollectorIT {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java
index ea5f32f..4830189 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java
@@ -22,6 +22,7 @@
import java.util.Map;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.stats.StatsCollectorIT;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
import org.junit.runners.Parameterized.Parameters;
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
similarity index 85%
rename from phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
rename to phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
index da8e78d..c424f45 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/stats/StatsCollectorIT.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.phoenix.end2end;
+package org.apache.phoenix.schema.stats;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE;
@@ -41,13 +41,19 @@
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
+import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
+import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.query.QueryServices;
@@ -55,9 +61,7 @@
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.schema.stats.GuidePostsInfo;
-import org.apache.phoenix.schema.stats.GuidePostsKey;
-import org.apache.phoenix.schema.stats.StatisticsUtil;
+import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -82,6 +86,7 @@
private String physicalTableName;
private final boolean userTableNamespaceMapped;
private final boolean mutable;
+ private static final int defaultGuidePostWidth = 20;
protected StatsCollectorIT(boolean mutable, boolean transactional, boolean userTableNamespaceMapped, boolean columnEncoded) {
StringBuilder sb = new StringBuilder();
@@ -119,10 +124,10 @@
// enable name space mapping at global level on both client and server side
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(7);
serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
- serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(defaultGuidePostWidth));
Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
- clientProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ clientProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(defaultGuidePostWidth));
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator()));
}
@@ -731,4 +736,97 @@
}
}
+ @Test
+ public void testGuidePostWidthUsedInDefaultStatsCollector() throws Exception {
+ String baseTable = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String ddl =
+ "CREATE TABLE " + baseTable
+ + " (k INTEGER PRIMARY KEY, a bigint, b bigint, c bigint) "
+ + tableDDLOptions;
+ BaseTest.createTestTable(getUrl(), ddl, null, null);
+ conn.createStatement().execute("upsert into " + baseTable + " values (100,1,1,1)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (101,2,2,2)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (102,3,3,3)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (103,4,4,4)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (104,5,5,5)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (105,6,6,6)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (106,7,7,7)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (107,8,8,8)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (108,9,9,9)");
+ conn.createStatement().execute("upsert into " + baseTable + " values (109,10,10,10)");
+ conn.commit();
+ DefaultStatisticsCollector statsCollector = getDefaultStatsCollectorForTable(baseTable);
+ statsCollector.init();
+ assertEquals(defaultGuidePostWidth, statsCollector.getGuidePostDepth());
+
+ // ok let's create a global index now and see what guide post width is used for it
+ String globalIndex = "GI_" + generateUniqueName();
+ ddl = "CREATE INDEX " + globalIndex + " ON " + baseTable + " (a) INCLUDE (b) ";
+ conn.createStatement().execute(ddl);
+ statsCollector = getDefaultStatsCollectorForTable(globalIndex);
+ statsCollector.init();
+ assertEquals(defaultGuidePostWidth, statsCollector.getGuidePostDepth());
+
+ // let's check out local index too
+ String localIndex = "LI_" + generateUniqueName();
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + baseTable + " (b) INCLUDE (c) ";
+ conn.createStatement().execute(ddl);
+ // local indexes reside on the same table as base data table
+ statsCollector = getDefaultStatsCollectorForTable(baseTable);
+ statsCollector.init();
+ assertEquals(defaultGuidePostWidth, statsCollector.getGuidePostDepth());
+
+ // now let's create a view and an index on it and see what guide post width is used for
+ // it
+ String view = "V_" + generateUniqueName();
+ ddl = "CREATE VIEW " + view + " AS SELECT * FROM " + baseTable;
+ conn.createStatement().execute(ddl);
+ String viewIndex = "VI_" + generateUniqueName();
+ ddl = "CREATE INDEX " + viewIndex + " ON " + view + " (b)";
+ conn.createStatement().execute(ddl);
+ String viewIndexTableName = MetaDataUtil.getViewIndexTableName(baseTable);
+ statsCollector = getDefaultStatsCollectorForTable(viewIndexTableName);
+ statsCollector.init();
+ assertEquals(defaultGuidePostWidth, statsCollector.getGuidePostDepth());
+ /*
+ * Fantastic! Now let's change the guide post width of the base table. This should
+ * change the guide post width we are using in DefaultStatisticsCollector for all
+ * indexes too.
+ */
+ long newGpWidth = 500;
+ conn.createStatement()
+ .execute("ALTER TABLE " + baseTable + " SET GUIDE_POSTS_WIDTH=" + newGpWidth);
+
+ // base table
+ statsCollector = getDefaultStatsCollectorForTable(baseTable);
+ statsCollector.init();
+ assertEquals(newGpWidth, statsCollector.getGuidePostDepth());
+
+ // global index table
+ statsCollector = getDefaultStatsCollectorForTable(globalIndex);
+ statsCollector.init();
+ assertEquals(newGpWidth, statsCollector.getGuidePostDepth());
+
+ // view index table
+ statsCollector = getDefaultStatsCollectorForTable(viewIndexTableName);
+ statsCollector.init();
+ assertEquals(newGpWidth, statsCollector.getGuidePostDepth());
+ }
+ }
+
+ private DefaultStatisticsCollector getDefaultStatsCollectorForTable(String tableName)
+ throws Exception {
+ RegionCoprocessorEnvironment env = getRegionEnvrionment(tableName);
+ return (DefaultStatisticsCollector) StatisticsCollectorFactory
+ .createStatisticsCollector(env, tableName, System.currentTimeMillis(), null, null);
+ }
+
+ private RegionCoprocessorEnvironment getRegionEnvrionment(String tableName)
+ throws IOException, InterruptedException {
+ return (RegionCoprocessorEnvironment) getUtility()
+ .getRSForFirstRegionInTable(TableName.valueOf(tableName))
+ .getOnlineRegions(TableName.valueOf(tableName)).get(0).getCoprocessorHost()
+ .findCoprocessorEnvironment(UngroupedAggregateRegionObserver.class.getName());
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index 8844cc8..9c4bcce 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -18,6 +18,8 @@
package org.apache.phoenix.schema.stats;
import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -45,15 +47,22 @@
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
/**
@@ -75,6 +84,7 @@
private ImmutableBytesWritable currentRow;
private final long clientTimeStamp;
private final String tableName;
+ private final boolean isViewIndexTable;
DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family,
byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
@@ -95,6 +105,9 @@
// since there's no row representing those in SYSTEM.CATALOG.
if (MetaDataUtil.isViewIndex(tableName)) {
pName = MetaDataUtil.getViewIndexUserTableName(tableName);
+ isViewIndexTable = true;
+ } else {
+ isViewIndexTable = false;
}
ptableKey = SchemaUtil.getTableKeyFromFullName(pName);
this.clientTimeStamp = clientTimeStamp;
@@ -109,7 +122,7 @@
}
}
- private void initGuidepostDepth() throws IOException {
+ private void initGuidepostDepth() throws IOException, ClassNotFoundException, SQLException {
// First check is if guidepost info set on statement itself
if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) {
int guidepostPerRegion = 0;
@@ -135,6 +148,38 @@
if (!result.isEmpty()) {
Cell cell = result.listCells().get(0);
guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault());
+ } else if (!isViewIndexTable) {
+ /*
+ * The table we are collecting stats for is potentially a base table, or local
+ * index or a global index. For view indexes, we rely on the the guide post
+ * width column in the parent data table's metadata which we already tried
+ * retrieving above.
+ */
+ try (Connection conn =
+ QueryUtil.getConnectionOnServer(env.getConfiguration())) {
+ PTable table = PhoenixRuntime.getTable(conn, tableName);
+ if (table.getType() == PTableType.INDEX
+ && table.getIndexType() == IndexType.GLOBAL) {
+ /*
+ * For global indexes, we need to get the parentName first and then
+ * fetch guide post width configured for the parent table.
+ */
+ PName parentName = table.getParentName();
+ byte[] parentKey =
+ SchemaUtil.getTableKeyFromFullName(parentName.getString());
+ get = new Get(parentKey);
+ get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
+ result = htable.get(get);
+ if (!result.isEmpty()) {
+ Cell cell = result.listCells().get(0);
+ guidepostWidth =
+ PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(),
+ cell.getValueOffset(), SortOrder.getDefault());
+ }
+ }
+ }
+
}
} finally {
if (htable != null) {
@@ -318,7 +363,11 @@
@Override
public void init() throws IOException {
- initGuidepostDepth();
+ try {
+ initGuidepostDepth();
+ } catch (ClassNotFoundException | SQLException e) {
+ throw new IOException("Unable to initialize the guide post depth", e);
+ }
this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp, guidePostDepth);
}
@@ -331,4 +380,9 @@
return null;
}
+ @VisibleForTesting // Don't call this method anywhere else
+ public long getGuidePostDepth() {
+ return guidePostDepth;
+ }
+
}
