| CREATE TABLE kafka (contents STRING); |
| LOAD DATA LOCAL INPATH '../../data/files/text-en.txt' INTO TABLE kafka; |
| set mapred.reduce.tasks=1; |
| set hive.exec.reducers.max=1; |
| |
| SELECT ngrams(sentences(lower(contents)), 1, 100, 1000).estfrequency FROM kafka; |
| SELECT ngrams(sentences(lower(contents)), 2, 100, 1000).estfrequency FROM kafka; |
| SELECT ngrams(sentences(lower(contents)), 3, 100, 1000).estfrequency FROM kafka; |
| SELECT ngrams(sentences(lower(contents)), 4, 100, 1000).estfrequency FROM kafka; |
| SELECT ngrams(sentences(lower(contents)), 5, 100, 1000).estfrequency FROM kafka; |
| |
| DROP TABLE kafka; |