Add Jieba tokenizer support
diff --git a/Cargo.lock b/Cargo.lock index ccb6369..b5b06ce 100644 --- a/Cargo.lock +++ b/Cargo.lock
@@ -1,6 +1,12 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "aho-corasick" @@ -71,12 +77,43 @@ ] [[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] name = "bytemuck" version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -107,6 +144,15 @@ ] [[package]] +name = "cedarwood" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" +dependencies = [ + "smallvec", +] + +[[package]] name = "census" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -184,6 +230,52 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" + +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + +[[package]] name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -194,9 +286,9 @@ [[package]] name = "downcast-rs" -version = "1.2.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "either" @@ -211,6 +303,17 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] name = "errno" version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -246,18 +349,18 @@ [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "fs4" -version = "0.8.4" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 0.38.44", - "windows-sys 0.52.0", + "rustix", + "windows-sys 0.59.0", ] [[package]] @@ -267,6 +370,17 @@ checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "futures-task" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -279,6 +393,7 @@ checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -286,17 +401,6 @@ [[package]] name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" @@ -320,9 +424,9 @@ [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", @@ -330,34 +434,64 @@ ] [[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] name = "htmlescape" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] -name = "instant" -version = "0.1.13" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "include-flate" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23e233413926ef735f7d87024466cfda5a4b87467730846bd82ea7d504121347" dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "include-flate-codegen", + "include-flate-compress", +] + +[[package]] +name = "include-flate-codegen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7148f24ef8922cc0e5574ebb908729ccdd3a110c440a45165733fedadd9969" +dependencies = [ + "include-flate-compress", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "include-flate-compress" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74783a9ed407e844e99d5e7a57bd650acbfa124cf6e97ffd790ba59d8ab8e7ff" +dependencies = [ + "libflate", + "zstd", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", ] [[package]] name = "itertools" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -369,6 +503,30 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] +name = "jieba-macros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +dependencies = [ + "phf_codegen", +] + +[[package]] +name = "jieba-rs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +dependencies = [ + "bytecount", + "cedarwood", + "include-flate", + "jieba-macros", + "phf", + "regex", + "rustc-hash", +] + +[[package]] name = "jni" version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -379,7 +537,7 @@ "combine", "jni-sys 0.3.1", "log", - "thiserror", + "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] @@ -434,6 +592,12 @@ ] [[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] name = "levenshtein_automata" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -446,16 +610,28 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] -name = "libm" -version = "0.2.16" +name = "libflate" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +dependencies = [ + "adler32", + "crc32fast", + "dary_heap", + "libflate_lz77", + "no_std_io2", +] [[package]] -name = "linux-raw-sys" -version = "0.4.15" +name = "libflate_lz77" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "ff7a10e427698aef6eef269482776debfef63384d30f13aad39a1a95e0e098fd" +dependencies = [ + "hashbrown", + "no_std_io2", + "rle-decode-fast", +] [[package]] name = "linux-raw-sys" @@ -471,26 +647,25 @@ [[package]] name = "lru" -version = "0.12.5" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown", ] [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" [[package]] name = "measure_time" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" dependencies = [ - "instant", "log", ] @@ -522,6 +697,15 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -544,17 +728,6 @@ checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", ] [[package]] @@ -570,10 +743,19 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] -name = "ownedbytes" -version = "0.7.0" +name = "ordered-float" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" dependencies = [ "stable_deref_trait", ] @@ -587,8 +769,9 @@ "serde", "serde_json", "tantivy", + "tantivy-jieba", "tempfile", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -612,6 +795,45 @@ ] [[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -630,12 +852,35 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ - "zerocopy", + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -669,46 +914,6 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand", -] - -[[package]] name = "rayon" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -758,6 +963,12 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] +name = "rle-decode-fast" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" + +[[package]] name = "roaring" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -779,22 +990,9 @@ [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" @@ -805,7 +1003,7 @@ "bitflags", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -874,10 +1072,16 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] -name = "sketches-ddsketch" -version = "0.2.2" +name = "siphasher" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "sketches-ddsketch" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" dependencies = [ "serde", ] @@ -901,6 +1105,12 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] name = "syn" version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -913,18 +1123,20 @@ [[package]] name = "tantivy" -version = "0.22.1" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96599ea6fccd844fc833fed21d2eecac2e6a7c1afd9e044057391d78b1feb141" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" dependencies = [ "aho-corasick", "arc-swap", "base64", "bitpacking", + "bon", "byteorder", "census", "crc32fast", "crossbeam-channel", + "datasketches", "downcast-rs", "fastdivide", "fnv", @@ -937,7 +1149,6 @@ "lz4_flex", "measure_time", "memmap2", - "num_cpus", "once_cell", "oneshot", "rayon", @@ -956,26 +1167,27 @@ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror", + "thiserror 2.0.18", "time", + "typetag", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.6.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", @@ -989,9 +1201,9 @@ [[package]] name = "tantivy-common" -version = "0.7.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" dependencies = [ "async-trait", "byteorder", @@ -1012,20 +1224,37 @@ ] [[package]] -name = "tantivy-query-grammar" -version = "0.22.0" +name = "tantivy-jieba" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +checksum = "3392170e86f1c387170aba7d171a466ffdc98a8b55b006e19ac64b123a7b690a" dependencies = [ + "jieba-rs", + "lazy_static", + "tantivy-tokenizer-api", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", "nom", + "ordered-float", + "serde", + "serde_json", ] [[package]] name = "tantivy-sstable" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ + "futures-util", + "itertools", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -1034,20 +1263,19 @@ [[package]] name = "tantivy-stacker" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" dependencies = [ "murmurhash32", - "rand_distr", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.3.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" dependencies = [ "serde", ] @@ -1061,7 +1289,7 @@ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.4", + "rustix", "windows-sys 0.61.2", ] @@ -1071,7 +1299,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1086,6 +1323,17 @@ ] [[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "time" version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1116,6 +1364,36 @@ ] [[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typetag" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1150,12 +1428,6 @@ ] [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1210,16 +1482,6 @@ ] [[package]] -name = "web-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1267,15 +1529,6 @@ [[package]] name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" @@ -1420,26 +1673,6 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] -name = "zerocopy" -version = "0.8.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml index 8a2e210..037ba19 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/apache/paimon-full-text" -rust-version = "1.75" +rust-version = "1.86" [workspace.dependencies] anyhow = "1" @@ -14,6 +14,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" roaring = "0.11" -tantivy = "0.22" +tantivy = "0.26" +tantivy-jieba = "0.20" tempfile = "3" thiserror = "1"
diff --git a/README.md b/README.md index 733c7b7..1b08b12 100644 --- a/README.md +++ b/README.md
@@ -30,10 +30,10 @@ - `whitespace` - `raw` - `ngram` +- `jieba` Reserved for follow-up: -- `jieba` - stemming - built-in and custom stop-word filters - true seek-on-demand Tantivy directory instead of loading segment files into
diff --git a/core/Cargo.toml b/core/Cargo.toml index 1f07c33..3bf1235 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml
@@ -11,6 +11,7 @@ serde.workspace = true serde_json.workspace = true tantivy.workspace = true +tantivy-jieba.workspace = true tempfile.workspace = true thiserror.workspace = true
diff --git a/core/src/index.rs b/core/src/index.rs index 30e7f1f..482a93f 100644 --- a/core/src/index.rs +++ b/core/src/index.rs
@@ -19,6 +19,7 @@ SimpleTokenizer, TextAnalyzer, WhitespaceTokenizer, }; use tantivy::{DocId, DocSet, Index, Score, SegmentReader, TantivyDocument, TERMINATED}; +use tantivy_jieba::JiebaTokenizer; use tempfile::TempDir; #[derive(Clone, Debug, PartialEq)] @@ -164,11 +165,11 @@ let collector = FilterCollector::new( row_id_field, move |row_id: u64| filter.contains(row_id), - TopDocs::with_limit(limit), + TopDocs::with_limit(limit).order_by_score(), ); searcher.search(&tantivy_query, &collector)? } else { - searcher.search(&tantivy_query, &TopDocs::with_limit(limit))? + searcher.search(&tantivy_query, &TopDocs::with_limit(limit).order_by_score())? }; let mut row_ids = Vec::with_capacity(top_docs.len()); let mut scores = Vec::with_capacity(top_docs.len()); @@ -238,10 +239,6 @@ fn register_tokenizer(index: &mut Index, config: &TokenizerConfig) -> Result<()> { match config.tokenizer { TokenizerKind::Default if !needs_custom_default(config) => Ok(()), - TokenizerKind::Jieba => Err(FtIndexError::InvalidOption { - key: "tokenizer".to_string(), - message: "jieba tokenizer is not enabled in this first implementation".to_string(), - }), _ => { let analyzer = build_text_analyzer(config)?; index @@ -281,10 +278,9 @@ TextAnalyzer::builder(tokenizer).dynamic() } TokenizerKind::Jieba => { - return Err(FtIndexError::InvalidOption { - key: "tokenizer".to_string(), - message: "jieba tokenizer is not enabled in this first implementation".to_string(), - }) + let mut tokenizer = JiebaTokenizer::with_search_mode(config.jieba_search_mode); + tokenizer.set_ordinal_position_mode(config.jieba_ordinal_position); + TextAnalyzer::builder(tokenizer).dynamic() } }; builder = builder.filter_dynamic(RemoveLongFilter::limit(config.max_token_length));
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs index 65635b2..783877d 100644 --- a/core/src/tokenizer.rs +++ b/core/src/tokenizer.rs
@@ -20,6 +20,8 @@ pub ngram_min_gram: usize, pub ngram_max_gram: usize, pub ngram_prefix_only: bool, + pub jieba_search_mode: bool, + pub jieba_ordinal_position: bool, pub lower_case: bool, pub max_token_length: usize, pub ascii_folding: bool, @@ -37,6 +39,8 @@ ngram_min_gram: 2, ngram_max_gram: 2, ngram_prefix_only: false, + jieba_search_mode: true, + jieba_ordinal_position: true, lower_case: true, max_token_length: 40, ascii_folding: false, @@ -70,6 +74,12 @@ "ngram.prefix-only" => { config.ngram_prefix_only = parse_bool(key, value)?; } + "jieba.search-mode" => { + config.jieba_search_mode = parse_bool(key, value)?; + } + "jieba.ordinal-position" | "jieba.ordinal-position-mode" => { + config.jieba_ordinal_position = parse_bool(key, value)?; + } "lower-case" => { config.lower_case = parse_bool(key, value)?; }
diff --git a/core/tests/core_roundtrip.rs b/core/tests/core_roundtrip.rs index 00d66ec..aa38ac3 100644 --- a/core/tests/core_roundtrip.rs +++ b/core/tests/core_roundtrip.rs
@@ -1,8 +1,10 @@ use paimon_ftindex_core::io::{PosWriter, SliceReader}; use paimon_ftindex_core::{ FullTextIndexConfig, FullTextIndexReader, FullTextIndexWriter, FullTextQuery, MatchOperator, + TokenizerConfig, TokenizerKind, }; use roaring::RoaringTreemap; +use std::collections::HashMap; fn build_index() -> anyhow::Result<Vec<u8>> { let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?; @@ -142,6 +144,68 @@ } #[test] +fn jieba_tokenizer_searches_chinese_terms() -> anyhow::Result<()> { + let config = FullTextIndexConfig::new().tokenizer(TokenizerConfig { + tokenizer: TokenizerKind::Jieba, + ..TokenizerConfig::default() + }); + let mut writer = FullTextIndexWriter::new(config)?; + writer.add_document(20, "中华人民共和国人民大会堂")?; + writer.add_document(21, "北京大学支持全文检索")?; + + let mut bytes = Vec::new(); + writer.write(&mut PosWriter::new(&mut bytes))?; + + let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?; + let result = reader.search(FullTextQuery::match_query("中华", "text"), 10)?; + + assert_eq!(result.row_ids, vec![20]); + Ok(()) +} + +#[test] +fn jieba_tokenizer_supports_chinese_phrase_queries() -> anyhow::Result<()> { + let config = FullTextIndexConfig::new().tokenizer(TokenizerConfig { + tokenizer: TokenizerKind::Jieba, + jieba_ordinal_position: true, + ..TokenizerConfig::default() + }); + let mut writer = FullTextIndexWriter::new(config)?; + writer.add_document(30, "北京大学支持全文检索")?; + writer.add_document(31, "北京的大学很多")?; + + let mut bytes = Vec::new(); + writer.write(&mut PosWriter::new(&mut bytes))?; + + let mut reader = FullTextIndexReader::open(SliceReader::new(bytes))?; + let result = reader.search(FullTextQuery::phrase("北京大学", "text"), 10)?; + + assert_eq!(result.row_ids, vec![30]); + Ok(()) +} + +#[test] +fn tokenizer_options_parse_jieba_settings() -> anyhow::Result<()> { + let mut options = HashMap::new(); + options.insert("fulltext.tokenizer".to_string(), "jieba".to_string()); + options.insert( + "fulltext.jieba.search-mode".to_string(), + "false".to_string(), + ); + options.insert( + "fulltext.jieba.ordinal-position".to_string(), + "false".to_string(), + ); + + let config = TokenizerConfig::from_options(&options)?; + + assert_eq!(config.tokenizer, TokenizerKind::Jieba); + assert!(!config.jieba_search_mode); + assert!(!config.jieba_ordinal_position); + Ok(()) +} + +#[test] fn boost_query_requires_positive_match() -> anyhow::Result<()> { let mut writer = FullTextIndexWriter::new(FullTextIndexConfig::new())?; writer.add_document(1, "apache paimon")?;
diff --git a/docs/designs/2026-07-04-paimon-full-text-index.md b/docs/designs/2026-07-04-paimon-full-text-index.md index 02e174f..ee22556 100644 --- a/docs/designs/2026-07-04-paimon-full-text-index.md +++ b/docs/designs/2026-07-04-paimon-full-text-index.md
@@ -106,7 +106,7 @@ } }, "document_count": 0, - "tantivy_version": "0.22.1" + "tantivy_version": "0.26.1" }, "files": [ {"name": "meta.json", "offset": 0, "length": 1234} @@ -367,6 +367,8 @@ fulltext.ngram.min-gram fulltext.ngram.max-gram fulltext.ngram.prefix-only +fulltext.jieba.search-mode +fulltext.jieba.ordinal-position fulltext.lower-case fulltext.max-token-length fulltext.ascii-folding @@ -389,10 +391,10 @@ - `whitespace` - `raw` - `ngram` +- `jieba` Reserved follow-up tokenizers and filters: -- `jieba` - stemming - built-in stop words - custom stop words @@ -403,6 +405,10 @@ - max token length - ASCII folding +The `jieba` tokenizer uses Tantivy's current external tokenizer API via +`tantivy-jieba`. Search mode and ordinal token positions are enabled by +default for Chinese full-text recall and phrase query correctness. + No dynamic Rust tokenizer plugins in v1. They complicate packaging and native loading too much for an initial library.
diff --git a/docs/paimon-integration.md b/docs/paimon-integration.md index e5da028..4905292 100644 --- a/docs/paimon-integration.md +++ b/docs/paimon-integration.md
@@ -24,6 +24,8 @@ fulltext.ngram.min-gram fulltext.ngram.max-gram fulltext.ngram.prefix-only +fulltext.jieba.search-mode +fulltext.jieba.ordinal-position fulltext.lower-case fulltext.max-token-length fulltext.ascii-folding