| (window.webpackJsonp=window.webpackJsonp||[]).push([[29],{593:function(t,e,r){"use strict";r.r(e);var a=r(68),s=Object(a.a)({},(function(){var t=this,e=t.$createElement,r=t._self._c||e;return r("ContentSlotsDistributor",{attrs:{"slot-key":t.$parent.slotKey}},[r("h1",{attrs:{id:"spark-tsfile-connector"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#spark-tsfile-connector"}},[t._v("#")]),t._v(" Spark Tsfile connector")]),t._v(" "),r("h2",{attrs:{id:"aim-of-design"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#aim-of-design"}},[t._v("#")]),t._v(" aim of design")]),t._v(" "),r("ul",[r("li",[r("p",[t._v("Use Spark SQL to read the data of the specified Tsfile and return it to the client in the form of a Spark DataFrame")])]),t._v(" "),r("li",[r("p",[t._v("Generate Tsfile with data from Spark Dataframe")])])]),t._v(" "),r("h2",{attrs:{id:"supported-formats"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#supported-formats"}},[t._v("#")]),t._v(" Supported formats")]),t._v(" "),r("p",[t._v("Wide table structure: Tsfile native format, IOTDB native path format")]),t._v(" "),r("table",[r("thead",[r("tr",[r("th",[t._v("time")]),t._v(" "),r("th",[t._v("root.ln.wf02.wt02.temperature")]),t._v(" "),r("th",[t._v("root.ln.wf02.wt02.status")]),t._v(" "),r("th",[t._v("root.ln.wf02.wt02.hardware")]),t._v(" "),r("th",[t._v("root.ln.wf01.wt01.temperature")]),t._v(" "),r("th",[t._v("root.ln.wf01.wt01.status")]),t._v(" "),r("th",[t._v("root.ln.wf01.wt01.hardware")])])]),t._v(" "),r("tbody",[r("tr",[r("td",[t._v("1")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("2.2")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("2")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("false")]),t._v(" "),r("td",[t._v("aaa")]),t._v(" "),r("td",[t._v("2.2")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("3")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("2.1")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("4")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("bbb")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("5")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("false")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("6")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("ccc")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")])])])]),t._v(" "),r("p",[t._v("Narrow table structure: Relational database schema, IOTDB align by device format")]),t._v(" "),r("table",[r("thead",[r("tr",[r("th",[t._v("time")]),t._v(" "),r("th",[t._v("device_name")]),t._v(" "),r("th",[t._v("status")]),t._v(" "),r("th",[t._v("hardware")]),t._v(" "),r("th",[t._v("temperature")])])]),t._v(" "),r("tbody",[r("tr",[r("td",[t._v("1")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("2.2")])]),t._v(" "),r("tr",[r("td",[t._v("1")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("2")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("2.2")])]),t._v(" "),r("tr",[r("td",[t._v("2")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),r("td",[t._v("false")]),t._v(" "),r("td",[t._v("aaa")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("3")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("2.1")])]),t._v(" "),r("tr",[r("td",[t._v("4")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),r("td",[t._v("true")]),t._v(" "),r("td",[t._v("bbb")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("5")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),r("td",[t._v("false")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("null")])]),t._v(" "),r("tr",[r("td",[t._v("6")]),t._v(" "),r("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),r("td",[t._v("null")]),t._v(" "),r("td",[t._v("ccc")]),t._v(" "),r("td",[t._v("null")])])])]),t._v(" "),r("h2",{attrs:{id:"query-process-steps"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#query-process-steps"}},[t._v("#")]),t._v(" Query process steps")]),t._v(" "),r("h4",{attrs:{id:"_1-table-structure-inference-and-generation"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#_1-table-structure-inference-and-generation"}},[t._v("#")]),t._v(" 1. Table structure inference and generation")]),t._v(" "),r("p",[t._v("This step is to make the table structure of the DataFrame match the table structure of the Tsfile to be queried.\nThe main logic is inferSchema function in src / main / scala / org / apache / iotdb / spark / tsfile / DefaultSource.scala")]),t._v(" "),r("h4",{attrs:{id:"_2-sql-parsing"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#_2-sql-parsing"}},[t._v("#")]),t._v(" 2. SQL parsing")]),t._v(" "),r("p",[t._v("The purpose of this step is to transform user SQL statements into Tsfile native query expressions.")]),t._v(" "),r("p",[t._v("The main logic is the buildReader function in src / main / scala / org / apache / iotdb / spark / tsfile / DefaultSource.scala. SQL parsing wide table structure and narrow table structure")]),t._v(" "),r("h4",{attrs:{id:"_3-wide-table-structure"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#_3-wide-table-structure"}},[t._v("#")]),t._v(" 3. Wide table structure")]),t._v(" "),r("p",[t._v("The main logic of the SQL analysis of the wide table structure is in src / main / scala / org / apache / iotdb / spark / tsfile / WideConverter.scala. This structure is basically the same as the Tsfile native query structure. No special processing is required, and the SQL statement is directly converted into Corresponding query expression")]),t._v(" "),r("h4",{attrs:{id:"_4-narrow-table-structure"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#_4-narrow-table-structure"}},[t._v("#")]),t._v(" 4. Narrow table structure")]),t._v(" "),r("p",[t._v("The main logic of the SQL analysis of the wide table structure is src / main / scala / org / apache / iotdb / spark / tsfile / NarrowConverter.scala.")]),t._v(" "),r("p",[t._v("Firstly we use required schema to decide which timeseries we should get from time file")]),t._v(" "),r("div",{staticClass:"language- line-numbers-mode"},[r("pre",{pre:!0,attrs:{class:"language-text"}},[r("code",[t._v("requiredSchema.foreach((field: StructField) => {\n if (field.name != QueryConstant.RESERVED_TIME\n && field.name != NarrowConverter.DEVICE_NAME) {\n measurementNames += field.name\n }\n})\n")])]),t._v(" "),r("div",{staticClass:"line-numbers-wrapper"},[r("span",{staticClass:"line-number"},[t._v("1")]),r("br"),r("span",{staticClass:"line-number"},[t._v("2")]),r("br"),r("span",{staticClass:"line-number"},[t._v("3")]),r("br"),r("span",{staticClass:"line-number"},[t._v("4")]),r("br"),r("span",{staticClass:"line-number"},[t._v("5")]),r("br"),r("span",{staticClass:"line-number"},[t._v("6")]),r("br")])]),r("p",[t._v("After the SQL is converted to an expression, the narrow table structure is different from the Tsfile native query structure. The expression is converted into a disjunction expression related to the device before it can be converted into a query of Tsfile. The conversion code is in src / main / java / org / apache / iotdb / spark / tsfile / qp")]),t._v(" "),r("p",[t._v("example：")]),t._v(" "),r("div",{staticClass:"language- line-numbers-mode"},[r("pre",{pre:!0,attrs:{class:"language-text"}},[r("code",[t._v("select time, device_name, s1 from tsfile_table where time > 1588953600000 and time < 1589040000000 and device_name = 'root.group1.d1'\n")])]),t._v(" "),r("div",{staticClass:"line-numbers-wrapper"},[r("span",{staticClass:"line-number"},[t._v("1")]),r("br")])]),r("p",[t._v("Obviously we only need timeseries 'root.group1.d1.s1' and our expression is [time > 1588953600000] and [time < 1589040000000]")]),t._v(" "),r("h4",{attrs:{id:"_5-query-execution"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#_5-query-execution"}},[t._v("#")]),t._v(" 5. Query execution")]),t._v(" "),r("p",[t._v("The actual data query execution is performed by the Tsfile native component, see:")]),t._v(" "),r("ul",[r("li",[r("RouterLink",{attrs:{to:"/SystemDesign/TsFile/Read.html"}},[t._v("Tsfile native query process")])],1)]),t._v(" "),r("h2",{attrs:{id:"write-step-flow"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#write-step-flow"}},[t._v("#")]),t._v(" Write step flow")]),t._v(" "),r("p",[t._v("Writing is mainly to convert the data in the Dataframe structure into Tsfile's RowRecord, and write using Tsfile Writer")]),t._v(" "),r("h4",{attrs:{id:"wide-table-structure"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#wide-table-structure"}},[t._v("#")]),t._v(" Wide table structure")]),t._v(" "),r("p",[t._v("The main conversion code is in the following two files:")]),t._v(" "),r("ul",[r("li",[r("p",[t._v("src/main/scala/org/apache/iotdb/spark/tsfile/WideConverter.scala responsible for structural transformation")])]),t._v(" "),r("li",[r("p",[t._v("src/main/scala/org/apache/iotdb/spark/tsfile/WideTsFileOutputWriter.scala responsible for matching the spark interface and performing writes, which will call the structure conversion function in the previous file")])])]),t._v(" "),r("h4",{attrs:{id:"narrow-table-structure"}},[r("a",{staticClass:"header-anchor",attrs:{href:"#narrow-table-structure"}},[t._v("#")]),t._v(" Narrow table structure")]),t._v(" "),r("p",[t._v("The main conversion code is in the following two files:")]),t._v(" "),r("ul",[r("li",[r("p",[t._v("src/main/scala/org/apache/iotdb/spark/tsfile/NarrowConverter.scala responsible for structural transformation")])]),t._v(" "),r("li",[r("p",[t._v("src/main/scala/org/apache/iotdb/spark/tsfile/NarrowTsFileOutputWriter.scala responsible for matching the spark interface and performing writes, which will call the structure conversion function in the previous file")])])])])}),[],!1,null,null,null);e.default=s.exports}}]); |