| (window.webpackJsonp=window.webpackJsonp||[]).push([[76],{638:function(t,a,s){"use strict";s.r(a);var e=s(68),n=Object(e.a)({},(function(){var t=this,a=t.$createElement,s=t._self._c||a;return s("ContentSlotsDistributor",{attrs:{"slot-key":t.$parent.slotKey}},[s("h1",{attrs:{id:"tsfile-spark-connector-user-guide"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#tsfile-spark-connector-user-guide"}},[t._v("#")]),t._v(" TsFile-Spark-Connector User Guide")]),t._v(" "),s("h2",{attrs:{id:"_1-about-tsfile-spark-connector"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_1-about-tsfile-spark-connector"}},[t._v("#")]),t._v(" 1. About TsFile-Spark-Connector")]),t._v(" "),s("p",[t._v("TsFile-Spark-Connector implements the support of Spark for external data sources of Tsfile type. This enables users to read, write and query Tsfile by Spark.")]),t._v(" "),s("p",[t._v("With this connector, you can")]),t._v(" "),s("ul",[s("li",[t._v("load a single TsFile, from either the local file system or hdfs, into Spark")]),t._v(" "),s("li",[t._v("load all files in a specific directory, from either the local file system or hdfs, into Spark")]),t._v(" "),s("li",[t._v("write data from Spark into TsFile")])]),t._v(" "),s("h2",{attrs:{id:"_2-system-requirements"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_2-system-requirements"}},[t._v("#")]),t._v(" 2. System Requirements")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",{staticStyle:{"text-align":"center"}},[t._v("Spark Version")]),t._v(" "),s("th",{staticStyle:{"text-align":"center"}},[t._v("Scala Version")]),t._v(" "),s("th",{staticStyle:{"text-align":"center"}},[t._v("Java Version")]),t._v(" "),s("th",{staticStyle:{"text-align":"center"}},[t._v("TsFile")])])]),t._v(" "),s("tbody",[s("tr",[s("td",{staticStyle:{"text-align":"center"}},[s("code",[t._v("2.4.3")])]),t._v(" "),s("td",{staticStyle:{"text-align":"center"}},[s("code",[t._v("2.11.8")])]),t._v(" "),s("td",{staticStyle:{"text-align":"center"}},[s("code",[t._v("1.8")])]),t._v(" "),s("td",{staticStyle:{"text-align":"center"}},[s("code",[t._v("0.10.0")])])])])]),t._v(" "),s("blockquote",[s("p",[t._v("Note: For more information about how to download and use TsFile, please see the following link: https://github.com/apache/iotdb/tree/master/tsfile.")])]),t._v(" "),s("h2",{attrs:{id:"_3-quick-start"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_3-quick-start"}},[t._v("#")]),t._v(" 3. Quick Start")]),t._v(" "),s("h3",{attrs:{id:"local-mode"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#local-mode"}},[t._v("#")]),t._v(" Local Mode")]),t._v(" "),s("p",[t._v("Start Spark with TsFile-Spark-Connector in local mode:")]),t._v(" "),s("div",{staticClass:"language- line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-text"}},[s("code",[t._v("./<spark-shell-path> --jars tsfile-spark-connector.jar,tsfile-0.10.0-jar-with-dependencies.jar\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br")])]),s("p",[t._v("Note:")]),t._v(" "),s("ul",[s("li",[t._v("<spark-shell-path> is the real path of your spark-shell.")]),t._v(" "),s("li",[t._v("Multiple jar packages are separated by commas without any spaces.")]),t._v(" "),s("li",[t._v("See https://github.com/apache/iotdb/tree/master/tsfile for how to get TsFile.")])]),t._v(" "),s("h3",{attrs:{id:"distributed-mode"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#distributed-mode"}},[t._v("#")]),t._v(" Distributed Mode")]),t._v(" "),s("p",[t._v("Start Spark with TsFile-Spark-Connector in distributed mode (That is, the spark cluster is connected by spark-shell):")]),t._v(" "),s("div",{staticClass:"language- line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-text"}},[s("code",[t._v(". /<spark-shell-path> --jars tsfile-spark-connector.jar,tsfile-{version}-jar-with-dependencies.jar --master spark://ip:7077\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br")])]),s("p",[t._v("Note:")]),t._v(" "),s("ul",[s("li",[t._v("<spark-shell-path> is the real path of your spark-shell.")]),t._v(" "),s("li",[t._v("Multiple jar packages are separated by commas without any spaces.")]),t._v(" "),s("li",[t._v("See https://github.com/apache/iotdb/tree/master/tsfile for how to get TsFile.")])]),t._v(" "),s("h2",{attrs:{id:"_4-data-type-correspondence"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_4-data-type-correspondence"}},[t._v("#")]),t._v(" 4. Data Type Correspondence")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("TsFile data type")]),t._v(" "),s("th",[t._v("SparkSQL data type")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("BOOLEAN")]),t._v(" "),s("td",[t._v("BooleanType")])]),t._v(" "),s("tr",[s("td",[t._v("INT32")]),t._v(" "),s("td",[t._v("IntegerType")])]),t._v(" "),s("tr",[s("td",[t._v("INT64")]),t._v(" "),s("td",[t._v("LongType")])]),t._v(" "),s("tr",[s("td",[t._v("FLOAT")]),t._v(" "),s("td",[t._v("FloatType")])]),t._v(" "),s("tr",[s("td",[t._v("DOUBLE")]),t._v(" "),s("td",[t._v("DoubleType")])]),t._v(" "),s("tr",[s("td",[t._v("TEXT")]),t._v(" "),s("td",[t._v("StringType")])])])]),t._v(" "),s("h2",{attrs:{id:"_5-schema-inference"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_5-schema-inference"}},[t._v("#")]),t._v(" 5. Schema Inference")]),t._v(" "),s("p",[t._v("The way to display TsFile is dependent on the schema. Take the following TsFile structure as an example: There are three measurements in the TsFile schema: status, temperature, and hardware. The basic information of these three measurements is listed:")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("Name")]),t._v(" "),s("th",[t._v("Type")]),t._v(" "),s("th",[t._v("Encode")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("status")]),t._v(" "),s("td",[t._v("Boolean")]),t._v(" "),s("td",[t._v("PLAIN")])]),t._v(" "),s("tr",[s("td",[t._v("temperature")]),t._v(" "),s("td",[t._v("Float")]),t._v(" "),s("td",[t._v("RLE")])]),t._v(" "),s("tr",[s("td",[t._v("hardware")]),t._v(" "),s("td",[t._v("Text")]),t._v(" "),s("td",[t._v("PLAIN")])])])]),t._v(" "),s("p",[t._v("The existing data in the TsFile are:")]),t._v(" "),s("img",{attrs:{width:"519",alt:"ST 1",src:"https://user-images.githubusercontent.com/69114052/98197920-be9abc80-1f62-11eb-9efb-027f0590031c.png"}}),t._v(" "),s("p",[t._v("The corresponding SparkSQL table is:")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("time")]),t._v(" "),s("th",[t._v("root.ln.wf02.wt02.temperature")]),t._v(" "),s("th",[t._v("root.ln.wf02.wt02.status")]),t._v(" "),s("th",[t._v("root.ln.wf02.wt02.hardware")]),t._v(" "),s("th",[t._v("root.ln.wf01.wt01.temperature")]),t._v(" "),s("th",[t._v("root.ln.wf01.wt01.status")]),t._v(" "),s("th",[t._v("root.ln.wf01.wt01.hardware")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("false")]),t._v(" "),s("td",[t._v("aaa")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.1")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("bbb")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("false")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("ccc")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])])])]),t._v(" "),s("p",[t._v("You can also use narrow table form which as follows: (You can see part 6 about how to use narrow form)")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("time")]),t._v(" "),s("th",[t._v("device_name")]),t._v(" "),s("th",[t._v("status")]),t._v(" "),s("th",[t._v("hardware")]),t._v(" "),s("th",[t._v("temperature")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.2")])]),t._v(" "),s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.2")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("false")]),t._v(" "),s("td",[t._v("aaa")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.1")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("true")]),t._v(" "),s("td",[t._v("bbb")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt01")]),t._v(" "),s("td",[t._v("false")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("ccc")]),t._v(" "),s("td",[t._v("null")])])])]),t._v(" "),s("h2",{attrs:{id:"_6-scala-api"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#_6-scala-api"}},[t._v("#")]),t._v(" 6. Scala API")]),t._v(" "),s("p",[t._v("NOTE: Remember to assign necessary read and write permissions in advance.")]),t._v(" "),s("h3",{attrs:{id:"example-1-read-from-the-local-file-system"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-1-read-from-the-local-file-system"}},[t._v("#")]),t._v(" Example 1: read from the local file system")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" wide_df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \nwide_df"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" narrow_df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \nnarrow_df"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br"),s("span",{staticClass:"line-number"},[t._v("6")]),s("br")])]),s("h3",{attrs:{id:"example-2-read-from-the-hadoop-file-system"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-2-read-from-the-hadoop-file-system"}},[t._v("#")]),t._v(" Example 2: read from the hadoop file system")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" wide_df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \nwide_df"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" narrow_df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \nnarrow_df"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br"),s("span",{staticClass:"line-number"},[t._v("6")]),s("br")])]),s("h3",{attrs:{id:"example-3-read-from-a-specific-directory"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-3-read-from-a-specific-directory"}},[t._v("#")]),t._v(" Example 3: read from a specific directory")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/usr/hadoop"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br")])]),s("p",[t._v("Note 1: Global time ordering of all TsFiles in a directory is not supported now.")]),t._v(" "),s("p",[t._v("Note 2: Measurements of the same name should have the same schema.")]),t._v(" "),s("h3",{attrs:{id:"example-4-query-in-wide-form"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-4-query-in-wide-form"}},[t._v("#")]),t._v(" Example 4: query in wide form")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("createOrReplaceTempView"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("sql"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"select * from tsfile_table where `device_1.sensor_1`>0 and `device_1.sensor_2` < 22"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br")])]),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("createOrReplaceTempView"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("sql"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"select count(*) from tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br")])]),s("h3",{attrs:{id:"example-5-query-in-narrow-form"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-5-query-in-narrow-form"}},[t._v("#")]),t._v(" Example 5: query in narrow form")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("createOrReplaceTempView"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("sql"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v("\"select * from tsfile_table where device_name = 'root.ln.wf02.wt02' and temperature > 5\"")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br")])]),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("createOrReplaceTempView"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("sql"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"select count(*) from tsfile_table"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br")])]),s("h3",{attrs:{id:"example-6-write-in-wide-form"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-6-write-in-wide-form"}},[t._v("#")]),t._v(" Example 6: write in wide form")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token comment"}},[t._v("// we only support wide_form table to write")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("write"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/output"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/output"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br"),s("span",{staticClass:"line-number"},[t._v("6")]),s("br"),s("span",{staticClass:"line-number"},[t._v("7")]),s("br"),s("span",{staticClass:"line-number"},[t._v("8")]),s("br"),s("span",{staticClass:"line-number"},[t._v("9")]),s("br")])]),s("h2",{attrs:{id:"example-6-write-in-narrow-form"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#example-6-write-in-narrow-form"}},[t._v("#")]),t._v(" Example 6: write in narrow form")]),t._v(" "),s("div",{staticClass:"language-scala line-numbers-mode"},[s("pre",{pre:!0,attrs:{class:"language-scala"}},[s("code",[s("span",{pre:!0,attrs:{class:"token comment"}},[t._v("// we only support wide_form table to write")]),t._v("\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("import")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token namespace"}},[t._v("org"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("apache"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("iotdb"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")])]),t._v("_\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" df "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/test.tsfile"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v(" \ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\ndf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("write"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/output"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\n\n"),s("span",{pre:!0,attrs:{class:"token keyword"}},[t._v("val")]),t._v(" newDf "),s("span",{pre:!0,attrs:{class:"token operator"}},[t._v("=")]),t._v(" spark"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("read"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("tsfile"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v("(")]),s("span",{pre:!0,attrs:{class:"token string"}},[t._v('"hdfs://localhost:9000/output"')]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(",")]),t._v(" "),s("span",{pre:!0,attrs:{class:"token boolean"}},[t._v("true")]),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(")")]),t._v("\nnewDf"),s("span",{pre:!0,attrs:{class:"token punctuation"}},[t._v(".")]),t._v("show\n")])]),t._v(" "),s("div",{staticClass:"line-numbers-wrapper"},[s("span",{staticClass:"line-number"},[t._v("1")]),s("br"),s("span",{staticClass:"line-number"},[t._v("2")]),s("br"),s("span",{staticClass:"line-number"},[t._v("3")]),s("br"),s("span",{staticClass:"line-number"},[t._v("4")]),s("br"),s("span",{staticClass:"line-number"},[t._v("5")]),s("br"),s("span",{staticClass:"line-number"},[t._v("6")]),s("br"),s("span",{staticClass:"line-number"},[t._v("7")]),s("br"),s("span",{staticClass:"line-number"},[t._v("8")]),s("br"),s("span",{staticClass:"line-number"},[t._v("9")]),s("br")])]),s("h2",{attrs:{id:"appendix-a-old-design-of-schema-inference"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#appendix-a-old-design-of-schema-inference"}},[t._v("#")]),t._v(" Appendix A: Old Design of Schema Inference")]),t._v(" "),s("p",[t._v("The way to display TsFile is related to TsFile Schema. Take the following TsFile structure as an example: There are three measurements in the Schema of TsFile: status, temperature, and hardware. The basic info of these three Measurements is:")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("Name")]),t._v(" "),s("th",[t._v("Type")]),t._v(" "),s("th",[t._v("Encode")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("status")]),t._v(" "),s("td",[t._v("Boolean")]),t._v(" "),s("td",[t._v("PLAIN")])]),t._v(" "),s("tr",[s("td",[t._v("temperature")]),t._v(" "),s("td",[t._v("Float")]),t._v(" "),s("td",[t._v("RLE")])]),t._v(" "),s("tr",[s("td",[t._v("hardware")]),t._v(" "),s("td",[t._v("Text")]),t._v(" "),s("td",[t._v("PLAIN")])])])]),t._v(" "),s("p",[t._v("The existing data in the file are:")]),t._v(" "),s("img",{attrs:{width:"817",alt:"ST 2",src:"https://user-images.githubusercontent.com/69114052/98197948-cf4b3280-1f62-11eb-9c8c-c97d1adf032c.png"}}),t._v(" "),s("p",[t._v("A set of time-series data")]),t._v(" "),s("p",[t._v("There are two ways to show a set of time-series data:")]),t._v(" "),s("h4",{attrs:{id:"the-default-way"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#the-default-way"}},[t._v("#")]),t._v(" the default way")]),t._v(" "),s("p",[t._v("Two columns are created to store the full path of the device: time(LongType) and delta_object(StringType).")]),t._v(" "),s("ul",[s("li",[s("code",[t._v("time")]),t._v(" : Timestamp, LongType")]),t._v(" "),s("li",[s("code",[t._v("delta_object")]),t._v(" : Delta_object ID, StringType")])]),t._v(" "),s("p",[t._v("Next, a column is created for each Measurement to store the specific data. The SparkSQL table structure is:")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("time(LongType)")]),t._v(" "),s("th",[t._v("delta_object(StringType)")]),t._v(" "),s("th",[t._v("status(BooleanType)")]),t._v(" "),s("th",[t._v("temperature(FloatType)")]),t._v(" "),s("th",[t._v("hardware(StringType)")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"aaa"')])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("2.1")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("3.3")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.0")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"bbb"')])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"ccc"')])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("6.6")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("7")]),t._v(" "),s("td",[t._v("root.ln.wf01.wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("8")]),t._v(" "),s("td",[t._v("root.ln.wf02.wt02")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"ddd"')])]),t._v(" "),s("tr",[s("td",[t._v("8")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("8.8")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("9")]),t._v(" "),s("td",[t._v("root.sgcc.wf03.wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("9.9")]),t._v(" "),s("td",[t._v("null")])])])]),t._v(" "),s("h4",{attrs:{id:"unfold-delta-object-column"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#unfold-delta-object-column"}},[t._v("#")]),t._v(" unfold delta_object column")]),t._v(" "),s("p",[t._v('Expand the device column by "." into multiple columns, ignoring the root directory "root". Convenient for richer aggregation operations. To use this display way, the parameter "delta_object_name" is set in the table creation statement (refer to Example 5 in Section 5.1 of this manual), as in this example, parameter "delta_object_name" is set to "root.device.turbine". The number of path layers needs to be one-to-one. At this point, one column is created for each layer of the device path except the "root" layer. The column name is the name in the parameter and the value is the name of the corresponding layer of the device. Next, one column is created for each Measurement to store the specific data.')]),t._v(" "),s("p",[t._v("Then SparkSQL Table Structure is as follows:")]),t._v(" "),s("table",[s("thead",[s("tr",[s("th",[t._v("time(LongType)")]),t._v(" "),s("th",[t._v("group(StringType)")]),t._v(" "),s("th",[t._v("field(StringType)")]),t._v(" "),s("th",[t._v("device(StringType)")]),t._v(" "),s("th",[t._v("status(BooleanType)")]),t._v(" "),s("th",[t._v("temperature(FloatType)")]),t._v(" "),s("th",[t._v("hardware(StringType)")])])]),t._v(" "),s("tbody",[s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("1")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.2")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"aaa"')])]),t._v(" "),s("tr",[s("td",[t._v("2")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("2.1")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("3")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("3.3")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("2.0")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"bbb"')])]),t._v(" "),s("tr",[s("td",[t._v("4")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("False")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("5")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"ccc"')])]),t._v(" "),s("tr",[s("td",[t._v("6")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("6.6")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("7")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf01")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("True")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("8")]),t._v(" "),s("td",[t._v("ln")]),t._v(" "),s("td",[t._v("wf02")]),t._v(" "),s("td",[t._v("wt02")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v('"ddd"')])]),t._v(" "),s("tr",[s("td",[t._v("8")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("8.8")]),t._v(" "),s("td",[t._v("null")])]),t._v(" "),s("tr",[s("td",[t._v("9")]),t._v(" "),s("td",[t._v("sgcc")]),t._v(" "),s("td",[t._v("wf03")]),t._v(" "),s("td",[t._v("wt01")]),t._v(" "),s("td",[t._v("null")]),t._v(" "),s("td",[t._v("9.9")]),t._v(" "),s("td",[t._v("null")])])])]),t._v(" "),s("p",[t._v("TsFile-Spark-Connector displays one or more TsFiles as a table in SparkSQL By SparkSQL. It also allows users to specify a single directory or use wildcards to match multiple directories. If there are multiple TsFiles, the union of the measurements in all TsFiles will be retained in the table, and the measurement with the same name have the same data type by default. Note that if a situation with the same name but different data types exists, TsFile-Spark-Connector does not guarantee the correctness of the results.")]),t._v(" "),s("p",[t._v('The writing process is to write a DataFrame as one or more TsFiles. By default, two columns need to be included: time and delta_object. The rest of the columns are used as Measurement. If user wants to write the second table structure back to TsFile, user can set the "delta_object_name" parameter(refer to Section 5.1 of Section 5.1 of this manual).')]),t._v(" "),s("h2",{attrs:{id:"appendix-b-old-note"}},[s("a",{staticClass:"header-anchor",attrs:{href:"#appendix-b-old-note"}},[t._v("#")]),t._v(" Appendix B: Old Note")]),t._v(" "),s("p",[t._v("NOTE: Check the jar packages in the root directory of your Spark and replace libthrift-0.9.2.jar and libfb303-0.9.2.jar with libthrift-0.9.1.jar and libfb303-0.9.1.jar respectively.")])])}),[],!1,null,null,null);a.default=n.exports}}]); |