| (window.webpackJsonp=window.webpackJsonp||[]).push([[547],{1112:function(e,s,t){"use strict";t.r(s);var a=t(70),n=Object(a.a)({},(function(){var e=this,s=e.$createElement,t=e._self._c||s;return t("ContentSlotsDistributor",{attrs:{"slot-key":e.$parent.slotKey}},[t("h2",{attrs:{id:"hadoop-tsfile"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#hadoop-tsfile"}},[e._v("#")]),e._v(" Hadoop-TsFile")]),e._v(" "),t("p",[e._v("TsFile的Hadoop连接器实现了对Hadoop读取外部Tsfile类型的文件格式的支持。让用户可以使用Hadoop的map、reduce等操作对Tsfile文件进行读取、写入和查询。")]),e._v(" "),t("p",[e._v("有了这个连接器，用户可以")]),e._v(" "),t("ul",[t("li",[e._v("将单个Tsfile文件加载进Hadoop，不论文件是存储在本地文件系统或者是HDFS中")]),e._v(" "),t("li",[e._v("将某个特定目录下的所有文件加载进Hadoop，不论文件是存储在本地文件系统或者是HDFS中")]),e._v(" "),t("li",[e._v("将Hadoop处理完后的结果以Tsfile的格式保存")])]),e._v(" "),t("h3",{attrs:{id:"系统环境要求"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#系统环境要求"}},[e._v("#")]),e._v(" 系统环境要求")]),e._v(" "),t("table",[t("thead",[t("tr",[t("th",[e._v("Hadoop 版本")]),e._v(" "),t("th",[e._v("Java 版本")]),e._v(" "),t("th",[e._v("TsFile 版本")])])]),e._v(" "),t("tbody",[t("tr",[t("td",[t("code",[e._v("2.7.3")])]),e._v(" "),t("td",[t("code",[e._v("1.8")])]),e._v(" "),t("td",[t("code",[e._v("0.10.0+")])])])])]),e._v(" "),t("blockquote",[t("p",[e._v("注意：关于如何下载和使用Tsfile, 请参考以下链接: https://github.com/apache/iotdb/tree/master/tsfile.")])]),e._v(" "),t("h3",{attrs:{id:"数据类型对应关系"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#数据类型对应关系"}},[e._v("#")]),e._v(" 数据类型对应关系")]),e._v(" "),t("table",[t("thead",[t("tr",[t("th",[e._v("TsFile 数据类型")]),e._v(" "),t("th",[e._v("Hadoop writable")])])]),e._v(" "),t("tbody",[t("tr",[t("td",[e._v("BOOLEAN")]),e._v(" "),t("td",[e._v("BooleanWritable")])]),e._v(" "),t("tr",[t("td",[e._v("INT32")]),e._v(" "),t("td",[e._v("IntWritable")])]),e._v(" "),t("tr",[t("td",[e._v("INT64")]),e._v(" "),t("td",[e._v("LongWritable")])]),e._v(" "),t("tr",[t("td",[e._v("FLOAT")]),e._v(" "),t("td",[e._v("FloatWritable")])]),e._v(" "),t("tr",[t("td",[e._v("DOUBLE")]),e._v(" "),t("td",[e._v("DoubleWritable")])]),e._v(" "),t("tr",[t("td",[e._v("TEXT")]),e._v(" "),t("td",[e._v("Text")])])])]),e._v(" "),t("h3",{attrs:{id:"关于tsfinputformat的说明"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#关于tsfinputformat的说明"}},[e._v("#")]),e._v(" 关于TSFInputFormat的说明")]),e._v(" "),t("p",[e._v("TSFInputFormat继承了Hadoop中FileInputFormat类，重写了其中切片的方法。")]),e._v(" "),t("p",[e._v("目前的切片方法是根据每个ChunkGroup的中点的offset是否属于Hadoop所切片的startOffset和endOffset之间，来判断是否将该ChunkGroup放入此切片。")]),e._v(" "),t("p",[e._v("TSFInputFormat将tsfile中的数据以多个"),t("code",[e._v("MapWritable")]),e._v("记录的形式返回给用户。")]),e._v(" "),t("p",[e._v("假设我们想要从Tsfile中获得名为"),t("code",[e._v("d1")]),e._v("的设备的数据，该设备有三个传感器，名称分别为"),t("code",[e._v("s1")]),e._v(", "),t("code",[e._v("s2")]),e._v(", "),t("code",[e._v("s3")]),e._v("。")]),e._v(" "),t("p",[t("code",[e._v("s1")]),e._v("的类型是"),t("code",[e._v("BOOLEAN")]),e._v(", "),t("code",[e._v("s2")]),e._v("的类型是 "),t("code",[e._v("DOUBLE")]),e._v(", "),t("code",[e._v("s3")]),e._v("的类型是"),t("code",[e._v("TEXT")]),e._v(".")]),e._v(" "),t("p",[t("code",[e._v("MapWritable")]),e._v("的结构如下所示：")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v('{\n "time_stamp": 10000000,\n "device_id": d1,\n "s1": true,\n "s2": 3.14,\n "s3": "middle"\n}\n')])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br")])]),t("p",[e._v("在Hadoop的Map job中，你可以采用如下方法获得你想要的任何值")]),e._v(" "),t("p",[t("code",[e._v('mapwritable.get(new Text("s1"))')])]),e._v(" "),t("blockquote",[t("p",[e._v("注意: "),t("code",[e._v("MapWritable")]),e._v("中所有的键值类型都是"),t("code",[e._v("Text")]),e._v("。")])]),e._v(" "),t("h3",{attrs:{id:"使用示例"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#使用示例"}},[e._v("#")]),e._v(" 使用示例")]),e._v(" "),t("h4",{attrs:{id:"读示例-求和"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#读示例-求和"}},[e._v("#")]),e._v(" 读示例: 求和")]),e._v(" "),t("p",[e._v("首先，我们需要在TSFInputFormat中配置我们需要哪些数据")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v('// configure reading time enable\nTSFInputFormat.setReadTime(job, true); \n// configure reading deviceId enable\nTSFInputFormat.setReadDeviceId(job, true); \n// configure reading which deltaObjectIds\nString[] deviceIds = {"device_1"};\nTSFInputFormat.setReadDeviceIds(job, deltaObjectIds);\n// configure reading which measurementIds\nString[] measurementIds = {"sensor_1", "sensor_2", "sensor_3"};\nTSFInputFormat.setReadMeasurementIds(job, measurementIds);\n')])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br"),t("span",{staticClass:"line-number"},[e._v("8")]),t("br"),t("span",{staticClass:"line-number"},[e._v("9")]),t("br"),t("span",{staticClass:"line-number"},[e._v("10")]),t("br")])]),t("p",[e._v("然后，必须指定mapper和reducer输出的键和值类型")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("// set inputformat and outputformat\njob.setInputFormatClass(TSFInputFormat.class);\n// set mapper output key and value\njob.setMapOutputKeyClass(Text.class);\njob.setMapOutputValueClass(DoubleWritable.class);\n// set reducer output key and value\njob.setOutputKeyClass(Text.class);\njob.setOutputValueClass(DoubleWritable.class);\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br"),t("span",{staticClass:"line-number"},[e._v("8")]),t("br")])]),t("p",[e._v("接着，就可以编写包含具体的处理数据逻辑的"),t("code",[e._v("mapper")]),e._v("和"),t("code",[e._v("reducer")]),e._v("类了。")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v('public static class TSMapper extends Mapper<NullWritable, MapWritable, Text, DoubleWritable> {\n\n\t@Override\n\tprotected void map(NullWritable key, MapWritable value,\n\t Mapper<NullWritable, MapWritable, Text, DoubleWritable>.Context context)\n\t throws IOException, InterruptedException {\n\t\n\t Text deltaObjectId = (Text) value.get(new Text("device_id"));\n\t context.write(deltaObjectId, (DoubleWritable) value.get(new Text("sensor_3")));\n\t}\n}\n\npublic static class TSReducer extends Reducer<Text, DoubleWritable, Text, DoubleWritable> {\n\n\t@Override\n\tprotected void reduce(Text key, Iterable<DoubleWritable> values,\n\t Reducer<Text, DoubleWritable, Text, DoubleWritable>.Context context)\n\t throws IOException, InterruptedException {\n\t\n\t double sum = 0;\n\t for (DoubleWritable value : values) {\n\t sum = sum + value.get();\n\t }\n\t context.write(key, new DoubleWritable(sum));\n\t}\n}\n')])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br"),t("span",{staticClass:"line-number"},[e._v("8")]),t("br"),t("span",{staticClass:"line-number"},[e._v("9")]),t("br"),t("span",{staticClass:"line-number"},[e._v("10")]),t("br"),t("span",{staticClass:"line-number"},[e._v("11")]),t("br"),t("span",{staticClass:"line-number"},[e._v("12")]),t("br"),t("span",{staticClass:"line-number"},[e._v("13")]),t("br"),t("span",{staticClass:"line-number"},[e._v("14")]),t("br"),t("span",{staticClass:"line-number"},[e._v("15")]),t("br"),t("span",{staticClass:"line-number"},[e._v("16")]),t("br"),t("span",{staticClass:"line-number"},[e._v("17")]),t("br"),t("span",{staticClass:"line-number"},[e._v("18")]),t("br"),t("span",{staticClass:"line-number"},[e._v("19")]),t("br"),t("span",{staticClass:"line-number"},[e._v("20")]),t("br"),t("span",{staticClass:"line-number"},[e._v("21")]),t("br"),t("span",{staticClass:"line-number"},[e._v("22")]),t("br"),t("span",{staticClass:"line-number"},[e._v("23")]),t("br"),t("span",{staticClass:"line-number"},[e._v("24")]),t("br"),t("span",{staticClass:"line-number"},[e._v("25")]),t("br"),t("span",{staticClass:"line-number"},[e._v("26")]),t("br")])]),t("blockquote",[t("p",[e._v("注意: 完整的代码示例可以在如下链接中找到：https://github.com/apache/iotdb/blob/master/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TSFMRReadExample.java")])]),e._v(" "),t("h3",{attrs:{id:"写示例-计算平均数并写入tsfile中"}},[t("a",{staticClass:"header-anchor",attrs:{href:"#写示例-计算平均数并写入tsfile中"}},[e._v("#")]),e._v(" 写示例: 计算平均数并写入Tsfile中")]),e._v(" "),t("p",[e._v("除了"),t("code",[e._v("OutputFormatClass")]),e._v("，剩下的配置代码跟上面的读示例是一样的")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v("job.setOutputFormatClass(TSFOutputFormat.class);\n// set reducer output key and value\njob.setOutputKeyClass(NullWritable.class);\njob.setOutputValueClass(HDFSTSRecord.class);\n")])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br")])]),t("p",[e._v("然后，是包含具体的处理数据逻辑的"),t("code",[e._v("mapper")]),e._v("和"),t("code",[e._v("reducer")]),e._v("类。")]),e._v(" "),t("div",{staticClass:"language- line-numbers-mode"},[t("pre",{pre:!0,attrs:{class:"language-text"}},[t("code",[e._v('public static class TSMapper extends Mapper<NullWritable, MapWritable, Text, MapWritable> {\n\n @Override\n protected void map(NullWritable key, MapWritable value,\n Mapper<NullWritable, MapWritable, Text, MapWritable>.Context context)\n throws IOException, InterruptedException {\n\n Text deltaObjectId = (Text) value.get(new Text("device_id"));\n long timestamp = ((LongWritable)value.get(new Text("timestamp"))).get();\n if (timestamp % 100000 == 0) {\n context.write(deltaObjectId, new MapWritable(value));\n }\n }\n}\n\n/**\n * This reducer calculate the average value.\n */\npublic static class TSReducer extends Reducer<Text, MapWritable, NullWritable, HDFSTSRecord> {\n\n @Override\n protected void reduce(Text key, Iterable<MapWritable> values,\n Reducer<Text, MapWritable, NullWritable, HDFSTSRecord>.Context context) throws IOException, InterruptedException {\n long sensor1_value_sum = 0;\n long sensor2_value_sum = 0;\n double sensor3_value_sum = 0;\n long num = 0;\n for (MapWritable value : values) {\n num++;\n sensor1_value_sum += ((LongWritable)value.get(new Text("sensor_1"))).get();\n sensor2_value_sum += ((LongWritable)value.get(new Text("sensor_2"))).get();\n sensor3_value_sum += ((DoubleWritable)value.get(new Text("sensor_3"))).get();\n }\n HDFSTSRecord tsRecord = new HDFSTSRecord(1L, key.toString());\n DataPoint dPoint1 = new LongDataPoint("sensor_1", sensor1_value_sum / num);\n DataPoint dPoint2 = new LongDataPoint("sensor_2", sensor2_value_sum / num);\n DataPoint dPoint3 = new DoubleDataPoint("sensor_3", sensor3_value_sum / num);\n tsRecord.addTuple(dPoint1);\n tsRecord.addTuple(dPoint2);\n tsRecord.addTuple(dPoint3);\n context.write(NullWritable.get(), tsRecord);\n }\n}\n')])]),e._v(" "),t("div",{staticClass:"line-numbers-wrapper"},[t("span",{staticClass:"line-number"},[e._v("1")]),t("br"),t("span",{staticClass:"line-number"},[e._v("2")]),t("br"),t("span",{staticClass:"line-number"},[e._v("3")]),t("br"),t("span",{staticClass:"line-number"},[e._v("4")]),t("br"),t("span",{staticClass:"line-number"},[e._v("5")]),t("br"),t("span",{staticClass:"line-number"},[e._v("6")]),t("br"),t("span",{staticClass:"line-number"},[e._v("7")]),t("br"),t("span",{staticClass:"line-number"},[e._v("8")]),t("br"),t("span",{staticClass:"line-number"},[e._v("9")]),t("br"),t("span",{staticClass:"line-number"},[e._v("10")]),t("br"),t("span",{staticClass:"line-number"},[e._v("11")]),t("br"),t("span",{staticClass:"line-number"},[e._v("12")]),t("br"),t("span",{staticClass:"line-number"},[e._v("13")]),t("br"),t("span",{staticClass:"line-number"},[e._v("14")]),t("br"),t("span",{staticClass:"line-number"},[e._v("15")]),t("br"),t("span",{staticClass:"line-number"},[e._v("16")]),t("br"),t("span",{staticClass:"line-number"},[e._v("17")]),t("br"),t("span",{staticClass:"line-number"},[e._v("18")]),t("br"),t("span",{staticClass:"line-number"},[e._v("19")]),t("br"),t("span",{staticClass:"line-number"},[e._v("20")]),t("br"),t("span",{staticClass:"line-number"},[e._v("21")]),t("br"),t("span",{staticClass:"line-number"},[e._v("22")]),t("br"),t("span",{staticClass:"line-number"},[e._v("23")]),t("br"),t("span",{staticClass:"line-number"},[e._v("24")]),t("br"),t("span",{staticClass:"line-number"},[e._v("25")]),t("br"),t("span",{staticClass:"line-number"},[e._v("26")]),t("br"),t("span",{staticClass:"line-number"},[e._v("27")]),t("br"),t("span",{staticClass:"line-number"},[e._v("28")]),t("br"),t("span",{staticClass:"line-number"},[e._v("29")]),t("br"),t("span",{staticClass:"line-number"},[e._v("30")]),t("br"),t("span",{staticClass:"line-number"},[e._v("31")]),t("br"),t("span",{staticClass:"line-number"},[e._v("32")]),t("br"),t("span",{staticClass:"line-number"},[e._v("33")]),t("br"),t("span",{staticClass:"line-number"},[e._v("34")]),t("br"),t("span",{staticClass:"line-number"},[e._v("35")]),t("br"),t("span",{staticClass:"line-number"},[e._v("36")]),t("br"),t("span",{staticClass:"line-number"},[e._v("37")]),t("br"),t("span",{staticClass:"line-number"},[e._v("38")]),t("br"),t("span",{staticClass:"line-number"},[e._v("39")]),t("br"),t("span",{staticClass:"line-number"},[e._v("40")]),t("br"),t("span",{staticClass:"line-number"},[e._v("41")]),t("br"),t("span",{staticClass:"line-number"},[e._v("42")]),t("br"),t("span",{staticClass:"line-number"},[e._v("43")]),t("br")])]),t("blockquote",[t("p",[e._v("注意: 完整的代码示例可以在如下链接中找到：https://github.com/apache/iotdb/blob/master/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TSMRWriteExample.java")])])])}),[],!1,null,null,null);s.default=n.exports}}]); |