| drop dataverse test if exists; | |
| create dataverse test; | |
| use dataverse test; | |
| write output to nc1:"rttest/tokenizers_hashed-word-tokens_01.adm"; | |
| let $txt := "Hello World, I would like to inform you of the importance of Foo Bar. Yes, Foo Bar. Jürgen." | |
| let $tokens := hashed-word-tokens($txt) | |
| for $token in $tokens | |
| return $token |