[CARBONDATA-3878] Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'
Why is this PR needed?
After the table has too many segments, file operation 'getLastModifiedTime' on all segment files will take a long time.
What changes were proposed in this PR?
Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'
Does this PR introduce any user interface change?
No
Is any new testcase added?
No
This closes #3814
diff --git a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
index 106042c..f38c33f 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
@@ -515,11 +515,16 @@
UpdateVO updateVO =
SegmentUpdateStatusManager.getInvalidTimestampRange(segment.getLoadMetadataDetails());
SegmentRefreshInfo segmentRefreshInfo;
- if (updateVO != null && updateVO.getLatestUpdateTimestamp() != null
+ if ((updateVO != null && updateVO.getLatestUpdateTimestamp() != null)
|| segment.getSegmentFileName() != null) {
- long segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
- .getSegmentFilePath(table.getTablePath(), segment.getSegmentFileName()))
- .getLastModifiedTime();
+ long segmentFileTimeStamp;
+ if (null != segment.getLoadMetadataDetails()) {
+ segmentFileTimeStamp = segment.getLoadMetadataDetails().getLastModifiedTime();
+ } else {
+ segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
+ .getSegmentFilePath(table.getTablePath(), segment.getSegmentFileName()))
+ .getLastModifiedTime();
+ }
segmentRefreshInfo =
new SegmentRefreshInfo(updateVO.getLatestUpdateTimestamp(), 0, segmentFileTimeStamp);
} else {
diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
index d6e3a62..4c73833 100644
--- a/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
+++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
@@ -95,7 +95,9 @@
public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) {
SegmentRefreshInfo segmentRefreshInfo;
long segmentFileTimeStamp = 0L;
- if (null != segment.getSegmentFileName()) {
+ if (null != segment.getLoadMetadataDetails()) {
+ segmentFileTimeStamp = segment.getLoadMetadataDetails().getLastModifiedTime();
+ } else if (null != segment.getSegmentFileName()) {
segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
.getSegmentFilePath(identifier.getTablePath(), segment.getSegmentFileName()))
.getLastModifiedTime();
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 0eef6af..7fb2cbd 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -487,4 +487,11 @@
fileFormat = null;
}
}
+
+ public long getLastModifiedTime() {
+ if (updateDeltaEndTimestamp != null) {
+ return convertTimeStampToLong(updateDeltaEndTimestamp);
+ }
+ return convertTimeStampToLong(timestamp);
+ }
}
