[CARBONDATA-3593] Add BLOCKLET_SCANNED_NUM and fix TOTAL_BLOCKLET_NUM not right when blocklet filtered
Add BLOCKLET_SCANNED_NUM and fix TOTAL_BLOCKLET_NUM not right when blocklet filtered.
This closes #3476
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index fa97d7a..25ba9a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -245,6 +245,11 @@
.put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
queryStatisticsModel.getRecorder().recordStatistics(queryStatisticValidScanBlocklet);
+ QueryStatistic scannedBlocklets = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM, scannedBlocklets);
+ queryStatisticsModel.getRecorder().recordStatistics(scannedBlocklets);
+
QueryStatistic totalNumberOfPages = new QueryStatistic();
queryStatisticsModel.getStatisticsTypeAndObjMap()
.put(QueryStatisticsConstants.TOTAL_PAGE_SCANNED, totalNumberOfPages);
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
index ca1dc15..347b9ce 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -113,6 +113,10 @@
.get(QueryStatisticsConstants.TOTAL_PAGE_SCANNED);
totalPagesScanned.addCountStatistic(QueryStatisticsConstants.TOTAL_PAGE_SCANNED,
totalPagesScanned.getCount() + dataBlock.numberOfPages());
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
// apply min max
if (isMinMaxEnabled) {
if (null == dataBlock.getColumnsMaxValue()
@@ -168,10 +172,6 @@
private BlockletScannedResult executeFilter(RawBlockletColumnChunks rawBlockletColumnChunks)
throws FilterUnsupportedException, IOException {
long startTime = System.currentTimeMillis();
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
// set the indexed data if it has any during fgdatamap pruning.
BitSetGroup fgBitSetGroup = rawBlockletColumnChunks.getDataBlock().getIndexedData();
rawBlockletColumnChunks.setBitSetGroup(fgBitSetGroup);
@@ -188,6 +188,11 @@
scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM);
+ scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM,
+ scannedBlocklets.getCount() + 1);
+
QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.PAGE_SCANNED);
scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
@@ -211,6 +216,12 @@
.get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
validPages.getCount() + bitSetGroup.getValidPages());
+
+ QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM);
+ scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM,
+ scannedBlocklets.getCount() + 1);
+
QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.PAGE_SCANNED);
scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
@@ -351,10 +362,6 @@
RawBlockletColumnChunks rawBlockletColumnChunks)
throws FilterUnsupportedException, IOException {
long startTime = System.currentTimeMillis();
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
// apply filter on actual data, for each page
BitSet pages = this.filterExecuter.prunePages(rawBlockletColumnChunks);
// if filter result is empty then return with empty result
@@ -367,6 +374,11 @@
scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
scanTime.getCount() + (System.currentTimeMillis() - startTime));
+ QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM);
+ scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM,
+ scannedBlocklets.getCount() + 1);
+
QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.PAGE_SCANNED);
scannedPages
@@ -388,6 +400,12 @@
.get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
validPages.getCount() + pages.cardinality());
+
+ QueryStatistic scannedBlocklets = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM);
+ scannedBlocklets.addCountStatistic(QueryStatisticsConstants.BLOCKLET_SCANNED_NUM,
+ scannedBlocklets.getCount() + 1);
+
QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
.get(QueryStatisticsConstants.PAGE_SCANNED);
scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
index 8a52294..31d9f93 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/QueryStatisticsConstants.java
@@ -52,6 +52,8 @@
String VALID_SCAN_BLOCKLET_NUM = "The num of valid scanned blocklet";
+ String BLOCKLET_SCANNED_NUM = "The num of blocklet scanned";
+
String VALID_PAGE_SCANNED = "The number of valid page scanned";
String TOTAL_PAGE_SCANNED = "The number of total page scanned";
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
index e63da18..2681f23 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/TaskStatistics.java
@@ -41,6 +41,7 @@
new Column("carbon_IO_time", QueryStatisticsConstants.READ_BLOCKlET_TIME),
new Column("scan_blocks_num", QueryStatisticsConstants.SCAN_BLOCKS_NUM),
new Column("total_blocklets", QueryStatisticsConstants.TOTAL_BLOCKLET_NUM),
+ new Column("scanned_blocklets", QueryStatisticsConstants.BLOCKLET_SCANNED_NUM),
new Column("valid_blocklets", QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM),
new Column("total_pages", QueryStatisticsConstants.TOTAL_PAGE_SCANNED),
new Column("scanned_pages", QueryStatisticsConstants.PAGE_SCANNED),
