)]}' { "commit": "580f38bc3522c2c823e5b1ea5f36dbd6afb335d4", "tree": "a11ba1435a3175eafc24944cb9ce11dbdf0a2868", "parents": [ "990f18fc8871199f561ff872276e604e291ae29d" ], "author": { "name": "星が降らない街", "email": "106635619+0130w@users.noreply.github.com", "time": "Mon Sep 23 16:13:32 2024 +0800" }, "committer": { "name": "morningman", "email": "morningman@163.com", "time": "Tue Sep 24 21:32:14 2024 +0800" }, "message": "[Feature](tvf) Support using tvf to read sequence_file/rc_file in local/hdfs/s3 (#41080)\n\nIssue Number: #30669\n\n\u003c!--Describe your changes.--\u003e\n\nThis change supports reading the contents of external file tables from\nrcbinary, rctext, and sequence files via the JNI connector.\n\ntodo-lists:\n- [x] Support read rc_binary files using local tvf\n- [x] Support read rc_text/sequence files using local tvf\n- [x] Support using s3/hdfs tvf\n\nExample:\n\n**sequence file:**\ninput:\n``` mysql\nselect * from local( \"file_path\" \u003d \"test/test.seq\", \"format\" \u003d \"sequence\", \"backend_id\" \u003d \"10011\", \"hive_schema\"\u003d\"k1:tinyint;k2:smallint;k3:int;k4:bigint;k5:float;k6:double;k7:decimal(10,2);k8:string;k9:char(10);k10:varchar(20);k11:boolean;k12:timestamp;k13:date;k14:array\u003cstring\u003e;k15:map\u003cstring,int\u003e;k16:struct\u003cname:string,age:int\u003e\");\n```\noutput:\n```\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n| k1 | k2 | k3 | k4 | k5 | k6 | k7 | k8 | k9 | k10 | k11 | k12 | k13 | k14 | k15 | k16 |\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n| 7 | 13 | 74 | 13000000000 | 6.15 | 4.376 | 57.30 | world | Char | Varchar | 1 | 2022-01-01 10:00:00 | 2022-01-01 | [\"A\", \"B\", \"C\"] | {\"key2\":2, \"key1\":1} | {\"name\":\"John\", \"age\":30} |\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n1 row in set (0.07 sec)\n```\n\n**rc_binary file:**\ninput:\n```mysql\nselect * from local( \"file_path\" \u003d \"test/test.rcbinary\", \"format\" \u003d \"rc_binary\", \"backend_id\" \u003d \"10011\", \"hive_schema\"\u003d\"k1:tinyint;k2:smallint;k3:int;k4:bigint;k5:float;k6:double;k7:decimal(10,2);k8:string;k9:char(10);k10:varchar(20);k11:boolean;k12:timestamp;k13:date;k14:array\u003cstring\u003e;k15:m\nap\u003cstring,int\u003e;k16:struct\u003cname:string,age:int\u003e\");\n```\noutput:\n```\n+------+------+------+-------------+------+------+--------+------+------------+-----------+------+---------------------+------------+-----------------+------------------+-------------------------------+\n| k1 | k2 | k3 | k4 | k5 | k6 | k7 | k8 | k9 | k10 | k11 | k12 | k13 | k14 | k15 | k16 |\n+------+------+------+-------------+------+------+--------+------+------------+-----------+------+---------------------+------------+-----------------+------------------+-------------------------------+\n| 1 | 2 | 3 | 10000000000 | 1.23 | 3.14 | 100.50 | you | are | beautiful | 0 | 2023-10-29 02:00:00 | 2023-10-29 | [\"D\", \"E\", \"F\"] | {\"k2\":5, \"k1\":3} | {\"name\":\"chandler\", \"age\":54} |\n+------+------+------+-------------+------+------+--------+------+------------+-----------+------+---------------------+------------+-----------------+------------------+-------------------------------+\n1 row in set (0.12 sec)\n```\n\n**rc_text file:**\ninput:\n``` mysql\nselect * from local( \"file_path\" \u003d \"test/test.rctext\", \"format\" \u003d \"rc_text\", \"backend_id\" \u003d \"10011\", \"hive_schema\"\u003d\"k1:tiny\nint;k2:smallint;k3:int;k4:bigint;k5:float;k6:double;k7:decimal(10,2);k8:string;k9:char(10);k10:varchar(20);k11:boolean;k12:timestamp;k13:date;k14:array\u003cstring\u003e;k15:\nmap\u003cstring,int\u003e;k16:struct\u003cname:string,age:int\u003e\");\n```\noutput:\n```\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n| k1 | k2 | k3 | k4 | k5 | k6 | k7 | k8 | k9 | k10 | k11 | k12 | k13 | k14 | k15 | k16 |\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n| 7 | 13 | 74 | 13000000000 | 6.15 | 4.376 | 57.30 | world | Char | Varchar | 1 | 2022-01-01 10:00:00 | 2022-01-01 | [\"A\", \"B\", \"C\"] | {\"key2\":2, \"key1\":1} | {\"name\":\"John\", \"age\":30} |\n+------+------+------+-------------+------+-------+-------+-------+------------+---------+------+---------------------+------------+-----------------+----------------------+---------------------------+\n1 row in set (0.06 sec)\n```\n", "tree_diff": [ { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "1b00cee1678be57d3873c734cc193a8a4979e1cb", "new_mode": 33188, "new_path": "be/src/vec/exec/format/hive/hive_jni_reader.cpp" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "14051c320e0915160d095cde4c06e820ef57a47b", "new_mode": 33188, "new_path": "be/src/vec/exec/format/hive/hive_jni_reader.h" }, { "type": "modify", "old_id": "0c2485ada3bbedbc7ba929510202f065f903fb74", "old_mode": 33188, "old_path": "be/src/vec/exec/jni_connector.cpp", "new_id": "1bba35a85dff1546099c4cb8b69adb9219ff8149", "new_mode": 33188, "new_path": "be/src/vec/exec/jni_connector.cpp" }, { "type": "modify", "old_id": "52a3fb2e7782ca6d1237ec056e4026f3f2a80988", "old_mode": 33188, "old_path": "be/src/vec/exec/jni_connector.h", "new_id": "7a1c5a1df4968c97824c19647723fd918c8fb8f2", "new_mode": 33188, "new_path": "be/src/vec/exec/jni_connector.h" }, { "type": "modify", "old_id": "52aa752935e88d0f47d3d8a24459ccbd5cde7939", "old_mode": 33188, "old_path": "be/src/vec/exec/scan/vfile_scanner.cpp", "new_id": "58f520e693baa8fce3dfc7771196dfba164b94e4", "new_mode": 33188, "new_path": "be/src/vec/exec/scan/vfile_scanner.cpp" }, { "type": "modify", "old_id": "1da5df76bb2fddd7051010d2ab3c33287d3bec07", "old_mode": 33261, "old_path": "build.sh", "new_id": "b7ac2ad7576a94d3090b7825660cf656c7b03a63", "new_mode": 33261, "new_path": "build.sh" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "64b4f7f0342b66e01c03993245a6b7115cf23efb", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/pom.xml" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "03c716677f7f61507a65f623a90d94685c7342e3", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/java/org/apache/doris/hive/HiveColumnValue.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "0b13935a5fcc2728eead01c3cb869cb8121202e6", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/java/org/apache/doris/hive/HiveFileContext.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "ae98bb28a10766a75699cd25307d7fb5221e614b", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/java/org/apache/doris/hive/HiveJNIScanner.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "3949c84dd314d72361bd102295e5c77ce7d52d64", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/java/org/apache/doris/hive/HiveProperties.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "45845af3c0392e325fc8d84754b1884df38702d5", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/java/org/apache/doris/hive/S3Utils.java" }, { "type": "add", "old_id": "0000000000000000000000000000000000000000", "old_mode": 0, "old_path": "/dev/null", "new_id": "d9ebe58202b9edb368c1a5a563100221176a3439", "new_mode": 33188, "new_path": "fe/be-java-extensions/hive-scanner/src/main/resources/package.xml" }, { "type": "modify", "old_id": "bbe056739d51eca2bb2599329d436e4d854f22ac", "old_mode": 33188, "old_path": "fe/be-java-extensions/pom.xml", "new_id": "8123c5c63d905cf99c3da78acca3c29d74283d33", "new_mode": 33188, "new_path": "fe/be-java-extensions/pom.xml" }, { "type": "modify", "old_id": "bdb2e97b9f25185cad6383ee1e4e425f10df2261", "old_mode": 33188, "old_path": "fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java", "new_id": "bdc12226600de140af266f93a4d8dfb746cfbafa", "new_mode": 33188, "new_path": "fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatConstants.java" }, { "type": "modify", "old_id": "15240f103b0e51a2d210e850edd41d24025dfcb5", "old_mode": 33188, "old_path": "fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatUtils.java", "new_id": "b5bc95f3173c59babeabe34d56f02b7cb7dcd954", "new_mode": 33188, "new_path": "fe/fe-core/src/main/java/org/apache/doris/common/util/FileFormatUtils.java" }, { "type": "modify", "old_id": "e137d5d200cc84243d54876cf33083eee4ad1da9", "old_mode": 33188, "old_path": "fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java", "new_id": "52d0477e581c86ac8b448149bac0a7699b480d8f", "new_mode": 33188, "new_path": "fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java" }, { "type": "modify", "old_id": "29b2b61530a935e2372c6cb6d13b36e71c991a09", "old_mode": 33188, "old_path": "fe/pom.xml", "new_id": "02150a03301b619cd55b035d46f15b33ae1ff75d", "new_mode": 33188, "new_path": "fe/pom.xml" }, { "type": "modify", "old_id": "426c3217918b71a65a99e9fc667ad00a92e2573c", "old_mode": 33188, "old_path": "gensrc/thrift/PlanNodes.thrift", "new_id": "b978616d6e5a3957ea175755c5b62f74aef85ad9", "new_mode": 33188, "new_path": "gensrc/thrift/PlanNodes.thrift" } ] }