(window.webpackJsonp=window.webpackJsonp||[]).push([[38],{605:function(e,a,t){"use strict";t.r(a);var i=t(70),s=Object(i.a)({},(function(){var e=this,a=e.$createElement,t=e._self._c||a;return t("ContentSlotsDistributor",{attrs:{"slot-key":e.$parent.slotKey}},[t("h1",{attrs:{id:"query-fundamentals"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#query-fundamentals"}},[e._v("#")]),e._v(" Query Fundamentals")]),e._v(" "),t("p",[e._v("This chapter introduces some basic concepts, terms and things to pay attention in IoTDB Query design.\nDesigners and developers who hope to start with IoTDB query design may find this guide helpful, as some concepts will be treated as common sense and not explained in detail in the following chapters.")]),e._v(" "),t("h2",{attrs:{id:"sequential-and-un-sequential-tsfiles"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#sequential-and-un-sequential-tsfiles"}},[e._v("#")]),e._v(" Sequential and un-sequential TsFiles")]),e._v(" "),t("p",[e._v("IoTDB uses TsFile as its data storage format. Sequential and un-sequential TsFiles are generated separately in terms of their different data insert patterns.\nBasically when time series data are written in strict ascending time order, only sequential TsFiles will be formed.\nAfter these sequential TsFiles are flushed onto disk, the current maximum timestamp of these sequential data will be recorded, and all the timeseries data with timestamps less than this maximum timestamp will be kept in un-sequential files.")]),e._v(" "),t("p",[e._v("IoTDB stores sequential and un-sequential TsFiles separately under "),t("code",[e._v("data/sequence")]),e._v(" and "),t("code",[e._v("data/unsequence")]),e._v(" directory. These files can be uniformly accessed via "),t("code",[e._v("getQueryDataSource()")]),e._v(" in "),t("code",[e._v("QueryResourceManager.java")]),e._v(", by giving a full path of the timeseries.")]),e._v(" "),t("p",[e._v("It should be noted that, in the following query documents, we tend to use "),t("code",[e._v("seq file")]),e._v(" to represent Sequential files for short. For un-sequential ones we use "),t("code",[e._v("unseq file")]),e._v(". Sometimes "),t("code",[e._v("unordered file")]),e._v(" or "),t("code",[e._v("out-of-order file")]),e._v(" are also used as aliases of un-sequential files.")]),e._v(" "),t("h2",{attrs:{id:"general-process-of-reading-tsfile-and-other-memory-data"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#general-process-of-reading-tsfile-and-other-memory-data"}},[e._v("#")]),e._v(" General process of reading TsFile and other memory data")]),e._v(" "),t("p",[e._v("The multi-level structure of TsFile is introduced in "),t("RouterLink",{attrs:{to:"/SystemDesign/TsFile/TsFile.html"}},[e._v("TsFile")]),e._v(".\nFor each timeseries, we always follow the query routine across 5 levels: TsFileResource -> TimeseriesMetadata -> ChunkMetadata -> IPageReader -> BatchData")],1),e._v(" "),t("p",[e._v("The function methods for file reading are in "),t("code",[e._v("org.apache.iotdb.db.utils.FileLoaderUtils")])]),e._v(" "),t("ul",[t("li",[t("code",[e._v("loadTimeSeriesMetadata()")]),e._v(" reads the TimeseriesMetadata of a timeseries in a TsFileResource. If a time filter is set for this method, only those TimeseriesMetadata that satisfies this filter will be returned. "),t("code",[e._v("loadTimeSeriesMetadata()")]),e._v(" returns null otherwise.")]),e._v(" "),t("li",[t("code",[e._v("loadChunkMetadataList()")]),e._v(" can load a ChunkMetadata list for a TimeseriesMetadata。")]),e._v(" "),t("li",[t("code",[e._v("loadPageReaderList()")]),e._v(" loads a page list contained in chunkMetadata，and can be accessed with "),t("code",[e._v("PageReader")]),e._v("。")])]),e._v(" "),t("p",[e._v("The implementation of the above methods must consider two cases of reading:")]),e._v(" "),t("ol",[t("li",[e._v("Reading memory data")]),e._v(" "),t("li",[e._v("Reading disk data.")])]),e._v(" "),t("p",[e._v('Memory data reading means to read data cached in "Memtable" which is not yet flushed into disk storage.\nIn '),t("code",[e._v("loadTimeSeriesMetadata()")]),e._v(", it obtains an unsealed TimeseriesMetadata using "),t("code",[e._v("TsFileResource.getTimeSeriesMetadata()")]),e._v(".\nWe call it unsealed because users may be still writing data into this Timeseries, and it will remain unsealed until IoTDB flushes it into disk.")]),e._v(" "),t("p",[t("code",[e._v("DiskChunkMetadataLoader")]),e._v(" and "),t("code",[e._v("MemChunkMetadataLoader")]),e._v(" provide access to read disk and memory chunk metadata.")]),e._v(" "),t("p",[e._v("It is almost the same in "),t("code",[e._v("loadPageReaderList()")]),e._v(" to read Page data.\n"),t("code",[e._v("MemChunkLoader")]),e._v(" and "),t("code",[e._v("DiskChunkLoader")]),e._v(" support for memory page loading and disk page loading.")]),e._v(" "),t("h2",{attrs:{id:"data-characteristics-of-sequential-and-unsequential-files"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#data-characteristics-of-sequential-and-unsequential-files"}},[e._v("#")]),e._v(" Data characteristics of sequential and unsequential files.")]),e._v(" "),t("p",[e._v('For data in sequential and unsequential files, the partial characteristics of the data in the file are different.\nTimeseries data in seq files is in "overall" ascending order. Specifically, all the ChunkMetadata in a timeseries stored in seq files are in the right order.\nTherefore, if we have '),t("code",[e._v("ChunkMetadata1")]),e._v(" and "),t("code",[e._v("ChunkMetadata2")]),e._v(" kept in a seq file, it is guaranteed that")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("chunkMetadata1.endtime <= chunkMetadata2.startTime.\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br")])]),t("p",[e._v("While it is different that the ChunkMetadatas are stored unordered in unseq files. Some chunks might be positioned in the right order but most of them are overlapped with each other. There might be overlapping between seq file chunks and unseq file chunks as well.")]),e._v(" "),t("p",[e._v("The page data in a single chunk is always sequential, no matter it is stored in seq files or unseq files.\nThat means, two orderings within pages are guaranteed:")]),e._v(" "),t("ul",[t("li",[e._v("Timestamps of data in a single page are in ascending order.")]),e._v(" "),t("li",[e._v("Different page timestamps are in ascending order. e.g. Page1.maxTime <= Page2.minTime.")])]),e._v(" "),t("p",[e._v("This certain ordering will be fully utilized to accelerate query process within our design.")]),e._v(" "),t("h2",{attrs:{id:"data-modification-processing-in-the-query"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#data-modification-processing-in-the-query"}},[e._v("#")]),e._v(" Data modification processing in the query")]),e._v(" "),t("p",[e._v("Data deletion in IoTDB records a series of mods file for disk data. The data is not really deleted, so we need to consider the existence of modifications in query.\nIf any data is deleted from a file, record the deletion to the mods file. Record three columns: the time series of the deletion, the maximum time point of the deletion range, and the version corresponding to the deletion operation.")]),e._v(" "),t("h3",{attrs:{id:"related-class"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#related-class"}},[e._v("#")]),e._v(" Related class")]),e._v(" "),t("p",[e._v("Modification file: org.apache.iotdb.db.engine.modification.ModificationFile")]),e._v(" "),t("p",[e._v("Deletion operation: org.apache.iotdb.db.engine.modification.Modification")]),e._v(" "),t("p",[e._v("Deletion interval: org.apache.iotdb.tsfile.read.common.TimeRange")]),e._v(" "),t("h3",{attrs:{id:"modification-file"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#modification-file"}},[e._v("#")]),e._v(" Modification File")]),e._v(" "),t("p",[e._v("Data deletion in IoTDB is accomplished by writing Modification files for related TsFiles.")]),e._v(" "),t("p",[e._v('In IoTDB version 0.11.0, the deletion format in Modification file has been changed. Now each line contains a start time and end time representing a delete range for a timeseries path.\nFor Modification files generated in past version of IoTDB with only a "deleteAt" timestamp, they could still be recognized, interpreting the "deleteAt" field as end time.')]),e._v(" "),t("h3",{attrs:{id:"timerange"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#timerange"}},[e._v("#")]),e._v(" TimeRange")]),e._v(" "),t("p",[e._v("Correspondingly, TimeRange is the medium that deletions exist within memory.")]),e._v(" "),t("p",[e._v("All deletion TimeRanges are both left-close and right-close intervals. We use Long.MIN_VALUE and Long.MAX_VALUE to refer to infinity and negative infinity timestamp.")]),e._v(" "),t("h3",{attrs:{id:"query-processing-with-deletion-ranges"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#query-processing-with-deletion-ranges"}},[e._v("#")]),e._v(" Query processing with deletion ranges")]),e._v(" "),t("p",[e._v("When querying a TVList, the TimeRanges are sorted and merged before a TVList tries to access them.\nFor example, we have [1,10], [5,12], [15,20], [16,21] in the original list, then they will be preprocessed to [1,12] and [15,21].\nFor cases when there are a large number of deletion operations, it would be helpful to exclude deleted data.")]),e._v(" "),t("p",[e._v("More specifically, since the TVList stores ordered timestamp data, using a sorted TimeRange list is easy to filter out deleted data.\nWe use a cursor to mark which TimeRange in the list is currently in use. Intervals before the current one are no longer needed to be traversed.")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("private boolean isPointDeleted(long timestamp) {\n  while (deletionList != null && deleteCursor < deletionList.size()) {\n    if (deletionList.get(deleteCursor).contains(timestamp)) {\n      return true;\n    } else if (deletionList.get(deleteCursor).getMax() < timestamp) {\n      deleteCursor++;\n    } else {\n      return false;\n    }\n  }\n  return false;\n}\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br"),t("span",{staticClass:"line-number"},[e._v("8")]),t("br"),t("span",{staticClass:"line-number"},[e._v("9")]),t("br"),t("span",{staticClass:"line-number"},[e._v("10")]),t("br"),t("span",{staticClass:"line-number"},[e._v("11")]),t("br"),t("span",{staticClass:"line-number"},[e._v("12")]),t("br")])]),t("h3",{attrs:{id:"query-with-modifications"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#query-with-modifications"}},[e._v("#")]),e._v(" Query with Modifications")]),e._v(" "),t("p",[e._v("For any TsFile data units, their metadata structures including TimeseriesMetadata, ChunkMetadata and PageHeader use a "),t("code",[e._v("modified")]),e._v(" flag to indicate whether this data unit is modified or not.\nUpon setting this "),t("code",[e._v("modified")]),e._v(' flag to "true", the integrity of this data unit is supposed to be damaged and some statistics turns invalid.')]),e._v(" "),t("p",[t("img",{attrs:{src:"https://user-images.githubusercontent.com/59866276/87266560-27fc4880-c4f8-11ea-9c8f-6794a9c599cb.jpg",alt:""}})]),e._v(" "),t("p",[e._v("Modifications affects timeseries reading process in the 5 levels mentioned before:")]),e._v(" "),t("ul",[t("li",[e._v("TsFileResource -> TimeseriesMetadata")])]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("// Set the statistics in TimeseriesMetadata unusable if the timeseries contains deletion operations \nFileLoaderUtils.loadTimeseriesMetadata()\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br")])]),t("ul",[t("li",[e._v("TimeseriesMetadata -> List<ChunkMetadata>")])]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("// For each ChunkMetadata, find the largest timestamp in all deletion operations whose version is larger than it. Set deleted time to ChunkMetadata. \n// set the statistics in ChunkMetadata unusable if it is affected by deletion\nFileLoaderUtils.loadChunkMetadataList()\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br")])]),t("p",[e._v("E.g., the got ChunkMetadatas are:\n"),t("img",{attrs:{src:"https://user-images.githubusercontent.com/59866276/87266976-0b144500-c4f9-11ea-95b3-15d60d2b7416.jpg",alt:""}})]),e._v(" "),t("ul",[t("li",[e._v("ChunkMetadata -> List<IPageReader>")])]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("// Skip the fully deleted page, set deleteAt into PageReader，Set the page statistics unusable if it is affected by deletion\nFileLoaderUtils.loadPageReaderList()\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br")])]),t("ul",[t("li",[e._v("IPageReader -> BatchData")])]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("// For disk page, skip the data points that be deleted and filtered out. For memory data, skip data points be filtered out.\nIPageReader.getAllSatisfiedPageData()\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br")])])])}),[],!1,null,null,null);a.default=s.exports}}]);