/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.analysis.email;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.EmojiTokenizationTestUnicode_11_0;
import org.apache.lucene.analysis.standard.WordBreakTestUnicode_9_0_0;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;

public class TestUAX29URLEmailTokenizer extends BaseTokenStreamTestCase {

  // LUCENE-5440: extremely slow tokenization of text matching email <local-part> (before the '@')
  @Nightly
  public void testLongEMAILatomText() throws Exception {
    // EMAILatomText = [A-Za-z0-9!#$%&'*+-/=?\^_`{|}~]
    char[] emailAtomChars =
        "!#$%&'*+,-./0123456789=?ABCDEFGHIJKLMNOPQRSTUVWXYZ^_`abcdefghijklmnopqrstuvwxyz{|}~"
            .toCharArray();
    StringBuilder builder = new StringBuilder();
    int numChars = TestUtil.nextInt(random(), 100 * 1024, 3 * 1024 * 1024);
    for (int i = 0; i < numChars; ++i) {
      builder.append(emailAtomChars[random().nextInt(emailAtomChars.length)]);
    }
    int tokenCount = 0;
    UAX29URLEmailTokenizer ts = new UAX29URLEmailTokenizer();
    String text = builder.toString();
    ts.setReader(new StringReader(text));
    ts.reset();
    while (ts.incrementToken()) {
      tokenCount++;
    }
    ts.end();
    ts.close();
    assertTrue(tokenCount > 0);

    tokenCount = 0;
    int newBufferSize = TestUtil.nextInt(random(), 200, 8192);
    ts.setMaxTokenLength(newBufferSize);
    ts.setReader(new StringReader(text));
    ts.reset();
    while (ts.incrementToken()) {
      tokenCount++;
    }
    ts.end();
    ts.close();
    assertTrue(tokenCount > 0);
  }

  public void testHugeDoc() throws IOException {
    StringBuilder sb = new StringBuilder();
    char whitespace[] = new char[4094];
    Arrays.fill(whitespace, ' ');
    sb.append(whitespace);
    sb.append("testing 1234");
    String input = sb.toString();
    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory());
    tokenizer.setReader(new StringReader(input));
    BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] {"testing", "1234"});
  }

  private Analyzer a, urlAnalyzer, emailAnalyzer;

  @Override
  public void setUp() throws Exception {
    super.setUp();
    a =
        new Analyzer() {
          @Override
          protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory());
            return new TokenStreamComponents(tokenizer);
          }
        };
    urlAnalyzer =
        new Analyzer() {
          @Override
          protected TokenStreamComponents createComponents(String fieldName) {
            UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory());
            tokenizer.setMaxTokenLength(
                UAX29URLEmailTokenizer.MAX_TOKEN_LENGTH_LIMIT); // Tokenize arbitrary length URLs
            TokenFilter filter = new URLFilter(tokenizer);
            return new TokenStreamComponents(tokenizer, filter);
          }
        };
    emailAnalyzer =
        new Analyzer() {
          @Override
          protected TokenStreamComponents createComponents(String fieldName) {
            UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(newAttributeFactory());
            TokenFilter filter = new EmailFilter(tokenizer);
            return new TokenStreamComponents(tokenizer, filter);
          }
        };
  }

  @Override
  public void tearDown() throws Exception {
    IOUtils.close(a, urlAnalyzer, emailAnalyzer);
    super.tearDown();
  }

  /** Passes through tokens with type "<URL>" and blocks all other types. */
  private static class URLFilter extends TokenFilter {
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);

    public URLFilter(TokenStream in) {
      super(in);
    }

    @Override
    public final boolean incrementToken() throws java.io.IOException {
      boolean isTokenAvailable = false;
      while (input.incrementToken()) {
        if (typeAtt.type() == UAX29URLEmailTokenizer.TOKEN_TYPES[UAX29URLEmailTokenizer.URL]) {
          isTokenAvailable = true;
          break;
        }
      }
      return isTokenAvailable;
    }
  }

  /** Passes through tokens with type "<EMAIL>" and blocks all other types. */
  private static class EmailFilter extends TokenFilter {
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);

    public EmailFilter(TokenStream in) {
      super(in);
    }

    @Override
    public final boolean incrementToken() throws java.io.IOException {
      boolean isTokenAvailable = false;
      while (input.incrementToken()) {
        if (typeAtt.type() == UAX29URLEmailTokenizer.TOKEN_TYPES[UAX29URLEmailTokenizer.EMAIL]) {
          isTokenAvailable = true;
          break;
        }
      }
      return isTokenAvailable;
    }
  }

  public void testArmenian() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",
        new String[] {
          "Վիքիպեդիայի",
          "13",
          "միլիոն",
          "հոդվածները",
          "4,600",
          "հայերեն",
          "վիքիպեդիայում",
          "գրվել",
          "են",
          "կամավորների",
          "կողմից",
          "ու",
          "համարյա",
          "բոլոր",
          "հոդվածները",
          "կարող",
          "է",
          "խմբագրել",
          "ցանկաց",
          "մարդ",
          "ով",
          "կարող",
          "է",
          "բացել",
          "Վիքիպեդիայի",
          "կայքը"
        });
  }

  public void testAmharic() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
        new String[] {
          "ዊኪፔድያ",
          "የባለ",
          "ብዙ",
          "ቋንቋ",
          "የተሟላ",
          "ትክክለኛና",
          "ነጻ",
          "መዝገበ",
          "ዕውቀት",
          "ኢንሳይክሎፒዲያ",
          "ነው",
          "ማንኛውም"
        });
  }

  public void testArabic() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",
        new String[] {
          "الفيلم",
          "الوثائقي",
          "الأول",
          "عن",
          "ويكيبيديا",
          "يسمى",
          "الحقيقة",
          "بالأرقام",
          "قصة",
          "ويكيبيديا",
          "بالإنجليزية",
          "Truth",
          "in",
          "Numbers",
          "The",
          "Wikipedia",
          "Story",
          "سيتم",
          "إطلاقه",
          "في",
          "2008"
        });
  }

  public void testAramaic() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",
        new String[] {
          "ܘܝܩܝܦܕܝܐ",
          "ܐܢܓܠܝܐ",
          "Wikipedia",
          "ܗܘ",
          "ܐܝܢܣܩܠܘܦܕܝܐ",
          "ܚܐܪܬܐ",
          "ܕܐܢܛܪܢܛ",
          "ܒܠܫܢ̈ܐ",
          "ܣܓܝܐ̈ܐ",
          "ܫܡܗ",
          "ܐܬܐ",
          "ܡܢ",
          "ܡ̈ܠܬܐ",
          "ܕ",
          "ܘܝܩܝ",
          "ܘ",
          "ܐܝܢܣܩܠܘܦܕܝܐ"
        });
  }

  public void testBengali() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
        new String[] {
          "এই",
          "বিশ্বকোষ",
          "পরিচালনা",
          "করে",
          "উইকিমিডিয়া",
          "ফাউন্ডেশন",
          "একটি",
          "অলাভজনক",
          "সংস্থা",
          "উইকিপিডিয়ার",
          "শুরু",
          "১৫",
          "জানুয়ারি",
          "২০০১",
          "সালে",
          "এখন",
          "পর্যন্ত",
          "২০০টিরও",
          "বেশী",
          "ভাষায়",
          "উইকিপিডিয়া",
          "রয়েছে"
        });
  }

  public void testFarsi() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",
        new String[] {
          "ویکی",
          "پدیای",
          "انگلیسی",
          "در",
          "تاریخ",
          "۲۵",
          "دی",
          "۱۳۷۹",
          "به",
          "صورت",
          "مکملی",
          "برای",
          "دانشنامهٔ",
          "تخصصی",
          "نوپدیا",
          "نوشته",
          "شد"
        });
  }

  public void testGreek() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",
        new String[] {
          "Γράφεται",
          "σε",
          "συνεργασία",
          "από",
          "εθελοντές",
          "με",
          "το",
          "λογισμικό",
          "wiki",
          "κάτι",
          "που",
          "σημαίνει",
          "ότι",
          "άρθρα",
          "μπορεί",
          "να",
          "προστεθούν",
          "ή",
          "να",
          "αλλάξουν",
          "από",
          "τον",
          "καθένα"
        });
  }

  public void testThai() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",
        new String[] {"การที่ได้ต้องแสดงว่างานดี", "แล้วเธอจะไปไหน", "๑๒๓๔"});
  }

  public void testLao() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ",
        new String[] {"ສາທາລະນະລັດ", "ປະຊາທິປະໄຕ", "ປະຊາຊົນລາວ"});
  }

  public void testTibetan() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
        new String[] {
          "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག",
          "མི", "ཉམས", "གོང", "འཕེལ", "དུ", "གཏོང", "བར",
          "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ"
        });
  }

  /*
   * For chinese, tokenize as char (these can later form bigrams or whatever)
   */
  public void testChinese() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "我是中国人。 １２３４ Ｔｅｓｔｓ ", new String[] {"我", "是", "中", "国", "人", "１２３４", "Ｔｅｓｔｓ"});
  }

  public void testEmpty() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "", new String[] {});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, ".", new String[] {});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, " ", new String[] {});
  }

  /* test various jira issues this analyzer is related to */

  public void testLUCENE1545() throws Exception {
    /*
     * Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTRE E.
     * The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.
     * Expected result is only on token "moͤchte".
     */
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "moͤchte", new String[] {"moͤchte"});
  }

  /* Tests from StandardAnalyzer, just to show behavior is similar */
  public void testAlphanumericSA() throws Exception {
    // alphanumeric tokens
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "B2B", new String[] {"B2B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "2B", new String[] {"2B"});
  }

  public void testDelimitersSA() throws Exception {
    // other delimiters: "-", "/", ","
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "some-dashed-phrase", new String[] {"some", "dashed", "phrase"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "dogs,chase,cats", new String[] {"dogs", "chase", "cats"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "ac/dc", new String[] {"ac", "dc"});
  }

  public void testApostrophesSA() throws Exception {
    // internal apostrophes: O'Reilly, you're, O'Reilly's
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly", new String[] {"O'Reilly"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "you're", new String[] {"you're"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "she's", new String[] {"she's"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "Jim's", new String[] {"Jim's"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "don't", new String[] {"don't"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "O'Reilly's", new String[] {"O'Reilly's"});
  }

  public void testNumericSA() throws Exception {
    // floating point, serial, model numbers, ip addresses, etc.
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "21.35", new String[] {"21.35"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "R2D2 C3PO", new String[] {"R2D2", "C3PO"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "216.239.63.104", new String[] {"216.239.63.104"});
  }

  public void testTextWithNumbersSA() throws Exception {
    // numbers
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "David has 5000 bones", new String[] {"David", "has", "5000", "bones"});
  }

  public void testVariousTextSA() throws Exception {
    // various
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "C embedded developers wanted", new String[] {"C", "embedded", "developers", "wanted"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "foo bar FOO BAR", new String[] {"foo", "bar", "FOO", "BAR"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "foo      bar .  FOO <> BAR", new String[] {"foo", "bar", "FOO", "BAR"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "\"QUOTED\" word", new String[] {"QUOTED", "word"});
  }

  public void testKoreanSA() throws Exception {
    // Korean words
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[] {"안녕하세요", "한글입니다"});
  }

  public void testOffsets() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "David has 5000 bones",
        new String[] {"David", "has", "5000", "bones"},
        new int[] {0, 6, 10, 15},
        new int[] {5, 9, 14, 20});
  }

  public void testTypes() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "David has 5000 bones",
        new String[] {"David", "has", "5000", "bones"},
        new String[] {"<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>"});
  }

  public void testWikiURLs() throws Exception {
    Reader reader = null;
    String luceneResourcesWikiPage;
    try {
      reader =
          new InputStreamReader(
              getClass().getResourceAsStream("LuceneResourcesWikiPage.html"),
              StandardCharsets.UTF_8);
      StringBuilder builder = new StringBuilder();
      char[] buffer = new char[1024];
      int numCharsRead;
      while (-1 != (numCharsRead = reader.read(buffer))) {
        builder.append(buffer, 0, numCharsRead);
      }
      luceneResourcesWikiPage = builder.toString();
    } finally {
      if (null != reader) {
        reader.close();
      }
    }
    assertTrue(null != luceneResourcesWikiPage && luceneResourcesWikiPage.length() > 0);
    BufferedReader bufferedReader = null;
    String[] urls;
    try {
      List<String> urlList = new ArrayList<>();
      bufferedReader =
          new BufferedReader(
              new InputStreamReader(
                  getClass().getResourceAsStream("LuceneResourcesWikiPageURLs.txt"),
                  StandardCharsets.UTF_8));
      String line;
      while (null != (line = bufferedReader.readLine())) {
        line = line.trim();
        if (line.length() > 0) {
          urlList.add(line);
        }
      }
      urls = urlList.toArray(new String[urlList.size()]);
    } finally {
      if (null != bufferedReader) {
        bufferedReader.close();
      }
    }
    assertTrue(null != urls && urls.length > 0);
    BaseTokenStreamTestCase.assertAnalyzesTo(urlAnalyzer, luceneResourcesWikiPage, urls);
  }

  public void testEmails() throws Exception {
    Reader reader = null;
    String randomTextWithEmails;
    try {
      reader =
          new InputStreamReader(
              getClass().getResourceAsStream("random.text.with.email.addresses.txt"),
              StandardCharsets.UTF_8);
      StringBuilder builder = new StringBuilder();
      char[] buffer = new char[1024];
      int numCharsRead;
      while (-1 != (numCharsRead = reader.read(buffer))) {
        builder.append(buffer, 0, numCharsRead);
      }
      randomTextWithEmails = builder.toString();
    } finally {
      if (null != reader) {
        reader.close();
      }
    }
    assertTrue(null != randomTextWithEmails && randomTextWithEmails.length() > 0);
    BufferedReader bufferedReader = null;
    String[] emails;
    try {
      List<String> emailList = new ArrayList<>();
      bufferedReader =
          new BufferedReader(
              new InputStreamReader(
                  getClass()
                      .getResourceAsStream(
                          "email.addresses.from.random.text.with.email.addresses.txt"),
                  StandardCharsets.UTF_8));
      String line;
      while (null != (line = bufferedReader.readLine())) {
        line = line.trim();
        if (line.length() > 0) {
          emailList.add(line);
        }
      }
      emails = emailList.toArray(new String[emailList.size()]);
    } finally {
      if (null != bufferedReader) {
        bufferedReader.close();
      }
    }
    assertTrue(null != emails && emails.length > 0);
    BaseTokenStreamTestCase.assertAnalyzesTo(emailAnalyzer, randomTextWithEmails, emails);
  }

  public void testMailtoSchemeEmails() throws Exception {
    // See LUCENE-3880
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "mailto:test@example.org",
        new String[] {"mailto", "test@example.org"},
        new String[] {"<ALPHANUM>", "<EMAIL>"});

    // TODO: Support full mailto: scheme URIs. See RFC 6068: http://tools.ietf.org/html/rfc6068
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "mailto:personA@example.com,personB@example.com?cc=personC@example.com"
            + "&subject=Subjectivity&body=Corpusivity%20or%20something%20like%20that",
        new String[] {
          "mailto",
          "personA@example.com",
          // TODO: recognize ',' address delimiter. Also, see examples of ';' delimiter use at:
          // http://www.mailto.co.uk/
          ",personB@example.com",
          "?cc=personC@example.com", // TODO: split field keys/values
          "subject",
          "Subjectivity",
          "body",
          "Corpusivity",
          "20or",
          "20something",
          "20like",
          "20that"
        }, // TODO: Hex decoding + re-tokenization
        new String[] {
          "<ALPHANUM>",
          "<EMAIL>",
          "<EMAIL>",
          "<EMAIL>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>",
          "<ALPHANUM>"
        });
  }

  public void testURLs() throws Exception {
    Reader reader = null;
    String randomTextWithURLs;
    try {
      reader =
          new InputStreamReader(
              getClass().getResourceAsStream("random.text.with.urls.txt"), StandardCharsets.UTF_8);
      StringBuilder builder = new StringBuilder();
      char[] buffer = new char[1024];
      int numCharsRead;
      while (-1 != (numCharsRead = reader.read(buffer))) {
        builder.append(buffer, 0, numCharsRead);
      }
      randomTextWithURLs = builder.toString();
    } finally {
      if (null != reader) {
        reader.close();
      }
    }
    assertTrue(null != randomTextWithURLs && randomTextWithURLs.length() > 0);
    BufferedReader bufferedReader = null;
    String[] urls;
    try {
      List<String> urlList = new ArrayList<>();
      bufferedReader =
          new BufferedReader(
              new InputStreamReader(
                  getClass().getResourceAsStream("urls.from.random.text.with.urls.txt"),
                  StandardCharsets.UTF_8));
      String line;
      while (null != (line = bufferedReader.readLine())) {
        line = line.trim();
        if (line.length() > 0) {
          urlList.add(line);
        }
      }
      urls = urlList.toArray(new String[urlList.size()]);
    } finally {
      if (null != bufferedReader) {
        bufferedReader.close();
      }
    }
    assertTrue(null != urls && urls.length > 0);
    BaseTokenStreamTestCase.assertAnalyzesTo(urlAnalyzer, randomTextWithURLs, urls);
  }

  public void testUnicodeWordBreaks() throws Exception {
    WordBreakTestUnicode_9_0_0 wordBreakTest = new WordBreakTestUnicode_9_0_0();
    wordBreakTest.test(a);
  }

  public void testSupplementary() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "𩬅艱鍟䇹愯瀛",
        new String[] {"𩬅", "艱", "鍟", "䇹", "愯", "瀛"},
        new String[] {
          "<IDEOGRAPHIC>",
          "<IDEOGRAPHIC>",
          "<IDEOGRAPHIC>",
          "<IDEOGRAPHIC>",
          "<IDEOGRAPHIC>",
          "<IDEOGRAPHIC>"
        });
  }

  public void testKorean() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "훈민정음", new String[] {"훈민정음"}, new String[] {"<HANGUL>"});
  }

  public void testJapanese() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "仮名遣い カタカナ",
        new String[] {"仮", "名", "遣", "い", "カタカナ"},
        new String[] {
          "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>"
        });
  }

  public void testCombiningMarks() throws Exception {
    checkOneTerm(a, "ざ", "ざ"); // hiragana
    checkOneTerm(a, "ザ", "ザ"); // katakana
    checkOneTerm(a, "壹゙", "壹゙"); // ideographic
    checkOneTerm(a, "아゙", "아゙"); // hangul
  }

  /**
   * Multiple consecutive chars in \p{Word_Break = MidLetter}, \p{Word_Break = MidNumLet}, and/or
   * \p{Word_Break = MidNum} should trigger a token split.
   */
  public void testMid() throws Exception {
    // ':' is in \p{WB:MidLetter}, which should trigger a split unless there is a Letter char on
    // both sides
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B", new String[] {"A:B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A::B", new String[] {"A", "B"});

    // '.' is in \p{WB:MidNumLet}, which should trigger a split unless there is a Letter or Numeric
    // char on both sides
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2", new String[] {"1.2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B", new String[] {"A.B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1..2", new String[] {"1", "2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A..B", new String[] {"A", "B"});

    // ',' is in \p{WB:MidNum}, which should trigger a split unless there is a Numeric char on both
    // sides
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2", new String[] {"1,2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,,2", new String[] {"1", "2"});

    // Mixed consecutive \p{WB:MidLetter} and \p{WB:MidNumLet} should trigger a split
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.:B", new String[] {"A", "B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:.B", new String[] {"A", "B"});

    // Mixed consecutive \p{WB:MidNum} and \p{WB:MidNumLet} should trigger a split
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,.2", new String[] {"1", "2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.,2", new String[] {"1", "2"});

    // '_' is in \p{WB:ExtendNumLet}

    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A:B", new String[] {"A:B_A:B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A:B_A::B", new String[] {"A:B_A", "B"});

    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1.2", new String[] {"1.2_1.2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A.B", new String[] {"A.B_A.B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1.2_1..2", new String[] {"1.2_1", "2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "A.B_A..B", new String[] {"A.B_A", "B"});

    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,2", new String[] {"1,2_1,2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "1,2_1,,2", new String[] {"1,2_1", "2"});

    BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A.:B", new String[] {"C_A", "B"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "C_A:.B", new String[] {"C_A", "B"});

    BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1,.2", new String[] {"3_1", "2"});
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "3_1.,2", new String[] {"3_1", "2"});
  }

  /** simple emoji */
  public void testEmoji() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "💩 💩💩",
        new String[] {"💩", "💩", "💩"},
        new String[] {"<EMOJI>", "<EMOJI>", "<EMOJI>"});
  }

  /** emoji zwj sequence */
  public void testEmojiSequence() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "👩‍❤️‍👩", new String[] {"👩‍❤️‍👩"}, new String[] {"<EMOJI>"});
  }

  /** emoji zwj sequence with fitzpatrick modifier */
  public void testEmojiSequenceWithModifier() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "👨🏼‍⚕️", new String[] {"👨🏼‍⚕️"}, new String[] {"<EMOJI>"});
  }

  /** regional indicator */
  public void testEmojiRegionalIndicator() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "🇺🇸🇺🇸", new String[] {"🇺🇸", "🇺🇸"}, new String[] {"<EMOJI>", "<EMOJI>"});
  }

  /** variation sequence */
  public void testEmojiVariationSequence() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "#️⃣", new String[] {"#️⃣"}, new String[] {"<EMOJI>"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "3️⃣",
        new String[] {
          "3️⃣",
        },
        new String[] {"<EMOJI>"});

    // text presentation sequences
    BaseTokenStreamTestCase.assertAnalyzesTo(a, "#\uFE0E", new String[] {}, new String[] {});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "3\uFE0E", // \uFE0E is included in \p{WB:Extend}
        new String[] {
          "3\uFE0E",
        },
        new String[] {"<NUM>"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "\u2B55\uFE0E", // \u2B55 = HEAVY BLACK CIRCLE
        new String[] {
          "\u2B55",
        },
        new String[] {"<EMOJI>"});
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "\u2B55\uFE0E\u200D\u2B55\uFE0E",
        new String[] {"\u2B55", "\u200D\u2B55"},
        new String[] {"<EMOJI>", "<EMOJI>"});
  }

  public void testEmojiTagSequence() throws Exception {
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a, "🏴󠁧󠁢󠁥󠁮󠁧󠁿", new String[] {"🏴󠁧󠁢󠁥󠁮󠁧󠁿"}, new String[] {"<EMOJI>"});
  }

  public void testEmojiTokenization() throws Exception {
    // simple emoji around latin
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "poo💩poo",
        new String[] {"poo", "💩", "poo"},
        new String[] {"<ALPHANUM>", "<EMOJI>", "<ALPHANUM>"});
    // simple emoji around non-latin
    BaseTokenStreamTestCase.assertAnalyzesTo(
        a,
        "💩中國💩",
        new String[] {"💩", "中", "國", "💩"},
        new String[] {"<EMOJI>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<EMOJI>"});
  }

  public void testUnicodeEmojiTests() throws Exception {
    EmojiTokenizationTestUnicode_11_0 emojiTest = new EmojiTokenizationTestUnicode_11_0();
    emojiTest.test(a);
  }

  /** blast some random strings through the analyzer */
  public void testRandomStrings() throws Exception {
    checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER);
  }

  /** blast some random large strings through the analyzer */
  public void testRandomHugeStrings() throws Exception {
    Random random = random();
    checkRandomData(random, a, 3 * RANDOM_MULTIPLIER, 8192);
  }

  public void testExampleURLs() throws Exception {
    String[] TLDs = {
      "aaa",
      "ac",
      "ai",
      "aarp",
      "abarth",
      "abb",
      "abbott",
      "abbvie",
      "abc",
      "able",
      "abogado",
      "abudhabi",
      "academy",
      "accenture",
      "accountant",
      "accountants",
      "aco",
      "active",
      "actor",
      "ad",
      "adac",
      "ads",
      "adult",
      "ae",
      "aeg",
      "aero",
      "aetna",
      "af",
      "afamilycompany",
      "afl",
      "africa",
      "ag",
      "agakhan",
      "agency",
      "aig",
      "aigo",
      "airbus",
      "airforce",
      "airtel",
      "akdn",
      "al",
      "alfaromeo",
      "alibaba",
      "alipay",
      "allfinanz",
      "allstate",
      "ally",
      "alsace",
      "alstom",
      "am",
      "americanexpress",
      "americanfamily",
      "amex",
      "amfam",
      "amica",
      "amsterdam",
      "analytics",
      "android",
      "anquan",
      "anz",
      "ao",
      "aol",
      "apartments",
      "app",
      "apple",
      "aq",
      "aquarelle",
      "ar",
      "arab",
      "aramco",
      "archi",
      "army",
      "arpa",
      "art",
      "arte",
      "as",
      "asda",
      "asia",
      "associates",
      "at",
      "athleta",
      "attorney",
      "au",
      "auction",
      "audi",
      "audible",
      "audio",
      "auspost",
      "author",
      "auto",
      "autos",
      "avianca",
      "aw",
      "aws",
      "ax",
      "axa",
      "az",
      "azure",
      "ba",
      "baby",
      "baidu",
      "banamex",
      "bananarepublic",
      "band",
      "bank",
      "bar",
      "barcelona",
      "barclaycard",
      "barclays",
      "barefoot",
      "bargains",
      "baseball",
      "basketball",
      "bauhaus",
      "bayern",
      "bb",
      "bbc",
      "bbt",
      "bbva",
      "bcg",
      "bcn",
      "bd",
      "be",
      "beats",
      "beauty",
      "beer",
      "bentley",
      "berlin",
      "best",
      "bestbuy",
      "bet",
      "bf",
      "bg",
      "bh",
      "bharti",
      "bi",
      "bible",
      "bid",
      "bike",
      "bing",
      "bingo",
      "bio",
      "biz",
      "bj",
      "black",
      "blackfriday",
      "blanco",
      "blockbuster",
      "blog",
      "bloomberg",
      "blue",
      "bm",
      "bms",
      "bmw",
      "bn",
      "bnl",
      "bnpparibas",
      "bo",
      "boats",
      "boehringer",
      "bofa",
      "bom",
      "bond",
      "boo",
      "book",
      "booking",
      "bosch",
      "bostik",
      "boston",
      "bot",
      "boutique",
      "box",
      "br",
      "bradesco",
      "bridgestone",
      "broadway",
      "broker",
      "brother",
      "brussels",
      "bs",
      "bt",
      "budapest",
      "bugatti",
      "build",
      "builders",
      "business",
      "buy",
      "buzz",
      "bv",
      "bw",
      "by",
      "bz",
      "bzh",
      "ca",
      "cab",
      "cafe",
      "cal",
      "call",
      "calvinklein",
      "cam",
      "camera",
      "camp",
      "cancerresearch",
      "canon",
      "capetown",
      "capital",
      "capitalone",
      "car",
      "caravan",
      "cards",
      "care",
      "career",
      "careers",
      "cars",
      "cartier",
      "casa",
      "case",
      "caseih",
      "cash",
      "casino",
      "cat",
      "catering",
      "catholic",
      "cba",
      "cbn",
      "cbre",
      "cbs",
      "cc",
      "cd",
      "ceb",
      "center",
      "ceo",
      "cern",
      "cf",
      "cfa",
      "cfd",
      "cg",
      "ch",
      "chanel",
      "channel",
      "chase",
      "chat",
      "cheap",
      "chintai",
      "christmas",
      "chrome",
      "chrysler",
      "church",
      "ci",
      "cipriani",
      "circle",
      "cisco",
      "citadel",
      "citi",
      "citic",
      "city",
      "cityeats",
      "ck",
      "cl",
      "claims",
      "cleaning",
      "click",
      "clinic",
      "clinique",
      "clothing",
      "cloud",
      "club",
      "clubmed",
      "cm",
      "cn",
      "co",
      "coach",
      "codes",
      "coffee",
      "college",
      "cologne",
      "com",
      "comcast",
      "commbank",
      "community",
      "company",
      "compare",
      "computer",
      "comsec",
      "condos",
      "construction",
      "consulting",
      "contact",
      "contractors",
      "cooking",
      "cookingchannel",
      "cool",
      "coop",
      "corsica",
      "country",
      "coupon",
      "coupons",
      "courses",
      "cr",
      "credit",
      "creditcard",
      "creditunion",
      "cricket",
      "crown",
      "crs",
      "cruise",
      "cruises",
      "csc",
      "cu",
      "cuisinella",
      "cv",
      "cw",
      "cx",
      "cy",
      "cymru",
      "cyou",
      "cz",
      "dabur",
      "dad",
      "dance",
      "data",
      "date",
      "dating",
      "datsun",
      "day",
      "dclk",
      "dds",
      "de",
      "deal",
      "dealer",
      "deals",
      "degree",
      "delivery",
      "dell",
      "deloitte",
      "delta",
      "democrat",
      "dental",
      "dentist",
      "desi",
      "design",
      "dev",
      "dhl",
      "diamonds",
      "diet",
      "digital",
      "direct",
      "directory",
      "discount",
      "discover",
      "dish",
      "diy",
      "dj",
      "dk",
      "dm",
      "dnp",
      "do",
      "docs",
      "doctor",
      "dodge",
      "dog",
      "doha",
      "domains",
      "dot",
      "download",
      "drive",
      "dtv",
      "dubai",
      "duck",
      "dunlop",
      "duns",
      "dupont",
      "durban",
      "dvag",
      "dvr",
      "dz",
      "earth",
      "eat",
      "ec",
      "eco",
      "edeka",
      "edu",
      "education",
      "ee",
      "eg",
      "email",
      "emerck",
      "energy",
      "engineer",
      "engineering",
      "enterprises",
      "epost",
      "epson",
      "equipment",
      "er",
      "ericsson",
      "erni",
      "es",
      "esq",
      "estate",
      "esurance",
      "et",
      "etisalat",
      "eu",
      "eurovision",
      "eus",
      "events",
      "everbank",
      "exchange",
      "expert",
      "exposed",
      "express",
      "extraspace",
      "fage",
      "fail",
      "fairwinds",
      "faith",
      "family",
      "fan",
      "fans",
      "farm",
      "farmers",
      "fashion",
      "fast",
      "fedex",
      "feedback",
      "ferrari",
      "ferrero",
      "fi",
      "fiat",
      "fidelity",
      "fido",
      "film",
      "final",
      "finance",
      "financial",
      "fire",
      "firestone",
      "firmdale",
      "fish",
      "fishing",
      "fit",
      "fitness",
      "fj",
      "fk",
      "flickr",
      "flights",
      "flir",
      "florist",
      "flowers",
      "fly",
      "fm",
      "fo",
      "foo",
      "food",
      "foodnetwork",
      "football",
      "ford",
      "forex",
      "forsale",
      "forum",
      "foundation",
      "fox",
      "fr",
      "free",
      "fresenius",
      "frl",
      "frogans",
      "frontdoor",
      "frontier",
      "ftr",
      "fujitsu",
      "fujixerox",
      "fun",
      "fund",
      "furniture",
      "futbol",
      "fyi",
      "ga",
      "gal",
      "gallery",
      "gallo",
      "gallup",
      "game",
      "games",
      "gap",
      "garden",
      "gb",
      "gbiz",
      "gd",
      "gdn",
      "ge",
      "gea",
      "gent",
      "genting",
      "george",
      "gf",
      "gg",
      "ggee",
      "gh",
      "gi",
      "gift",
      "gifts",
      "gives",
      "giving",
      "gl",
      "glade",
      "glass",
      "gle",
      "global",
      "globo",
      "gm",
      "gmail",
      "gmbh",
      "gmo",
      "gmx",
      "gn",
      "godaddy",
      "gold",
      "goldpoint",
      "golf",
      "goo",
      "goodhands",
      "goodyear",
      "goog",
      "google",
      "gop",
      "got",
      "gov",
      "gp",
      "gq",
      "gr",
      "grainger",
      "graphics",
      "gratis",
      "green",
      "gripe",
      "grocery",
      "group",
      "gs",
      "gt",
      "gu",
      "guardian",
      "gucci",
      "guge",
      "guide",
      "guitars",
      "guru",
      "gw",
      "gy",
      "hair",
      "hamburg",
      "hangout",
      "haus",
      "hbo",
      "hdfc",
      "hdfcbank",
      "health",
      "healthcare",
      "help",
      "helsinki",
      "here",
      "hermes",
      "hgtv",
      "hiphop",
      "hisamitsu",
      "hitachi",
      "hiv",
      "hk",
      "hkt",
      "hm",
      "hn",
      "hockey",
      "holdings",
      "holiday",
      "homedepot",
      "homegoods",
      "homes",
      "homesense",
      "honda",
      "honeywell",
      "horse",
      "hospital",
      "host",
      "hosting",
      "hot",
      "hoteles",
      "hotels",
      "hotmail",
      "house",
      "how",
      "hr",
      "hsbc",
      "ht",
      "hu",
      "hughes",
      "hyatt",
      "hyundai",
      "ibm",
      "icbc",
      "ice",
      "icu",
      "id",
      "ie",
      "ieee",
      "ifm",
      "ikano",
      "il",
      "im",
      "imamat",
      "imdb",
      "immo",
      "immobilien",
      "in",
      "industries",
      "infiniti",
      "info",
      "ing",
      "ink",
      "institute",
      "insurance",
      "insure",
      "int",
      "intel",
      "international",
      "intuit",
      "investments",
      "io",
      "ipiranga",
      "iq",
      "ir",
      "irish",
      "is",
      "iselect",
      "ismaili",
      "ist",
      "istanbul",
      "it",
      "itau",
      "itv",
      "iveco",
      "iwc",
      "jaguar",
      "java",
      "jcb",
      "jcp",
      "je",
      "jeep",
      "jetzt",
      "jewelry",
      "jio",
      "jlc",
      "jll",
      "jm",
      "jmp",
      "jnj",
      "jo",
      "jobs",
      "joburg",
      "jot",
      "joy",
      "jp",
      "jpmorgan",
      "jprs",
      "juegos",
      "juniper",
      "kaufen",
      "kddi",
      "ke",
      "kerryhotels",
      "kerrylogistics",
      "kerryproperties",
      "kfh",
      "kg",
      "kh",
      "ki",
      "kia",
      "kim",
      "kinder",
      "kindle",
      "kitchen",
      "kiwi",
      "km",
      "kn",
      "koeln",
      "komatsu",
      "kosher",
      "kp",
      "kpmg",
      "kpn",
      "kr",
      "krd",
      "kred",
      "kuokgroup",
      "kw",
      "ky",
      "kyoto",
      "kz",
      "la",
      "lacaixa",
      "ladbrokes",
      "lamborghini",
      "lamer",
      "lancaster",
      "lancia",
      "lancome",
      "land",
      "landrover",
      "lanxess",
      "lasalle",
      "lat",
      "latino",
      "latrobe",
      "law",
      "lawyer",
      "lb",
      "lc",
      "lds",
      "lease",
      "leclerc",
      "lefrak",
      "legal",
      "lego",
      "lexus",
      "lgbt",
      "li",
      "liaison",
      "lidl",
      "life",
      "lifeinsurance",
      "lifestyle",
      "lighting",
      "like",
      "lilly",
      "limited",
      "limo",
      "lincoln",
      "linde",
      "link",
      "lipsy",
      "live",
      "living",
      "lixil",
      "lk",
      "llc",
      "loan",
      "loans",
      "locker",
      "locus",
      "loft",
      "lol",
      "london",
      "lotte",
      "lotto",
      "love",
      "lpl",
      "lplfinancial",
      "lr",
      "ls",
      "lt",
      "ltd",
      "ltda",
      "lu",
      "lundbeck",
      "lupin",
      "luxe",
      "luxury",
      "lv",
      "ly",
      "ma",
      "macys",
      "madrid",
      "maif",
      "maison",
      "makeup",
      "man",
      "management",
      "mango",
      "map",
      "market",
      "marketing",
      "markets",
      "marriott",
      "marshalls",
      "maserati",
      "mattel",
      "mba",
      "mc",
      "mckinsey",
      "md",
      "me",
      "med",
      "media",
      "meet",
      "melbourne",
      "meme",
      "memorial",
      "men",
      "menu",
      "merckmsd",
      "metlife",
      "mg",
      "mh",
      "miami",
      "microsoft",
      "mil",
      "mini",
      "mint",
      "mit",
      "mitsubishi",
      "mk",
      "ml",
      "mlb",
      "mls",
      "mm",
      "mma",
      "mn",
      "mo",
      "mobi",
      "mobile",
      "mobily",
      "moda",
      "moe",
      "moi",
      "mom",
      "monash",
      "money",
      "monster",
      "mopar",
      "mormon",
      "mortgage",
      "moscow",
      "moto",
      "motorcycles",
      "mov",
      "movie",
      "movistar",
      "mp",
      "mq",
      "mr",
      "ms",
      "msd",
      "mt",
      "mtn",
      "mtr",
      "mu",
      "museum",
      "mutual",
      "mv",
      "mw",
      "mx",
      "my",
      "mz",
      "na",
      "nab",
      "nadex",
      "nagoya",
      "name",
      "nationwide",
      "natura",
      "navy",
      "nba",
      "nc",
      "ne",
      "nec",
      "net",
      "netbank",
      "netflix",
      "network",
      "neustar",
      "new",
      "newholland",
      "news",
      "next",
      "nextdirect",
      "nexus",
      "nf",
      "nfl",
      "ng",
      "ngo",
      "nhk",
      "ni",
      "nico",
      "nike",
      "nikon",
      "ninja",
      "nissan",
      "nissay",
      "nl",
      "no",
      "nokia",
      "northwesternmutual",
      "norton",
      "now",
      "nowruz",
      "nowtv",
      "np",
      "nr",
      "nra",
      "nrw",
      "ntt",
      "nu",
      "nyc",
      "nz",
      "obi",
      "observer",
      "off",
      "office",
      "okinawa",
      "olayan",
      "olayangroup",
      "oldnavy",
      "ollo",
      "om",
      "omega",
      "one",
      "ong",
      "onl",
      "online",
      "onyourside",
      "ooo",
      "open",
      "oracle",
      "orange",
      "org",
      "organic",
      "origins",
      "osaka",
      "otsuka",
      "ott",
      "ovh",
      "pa",
      "page",
      "panasonic",
      "panerai",
      "paris",
      "pars",
      "partners",
      "parts",
      "party",
      "passagens",
      "pay",
      "pccw",
      "pe",
      "pet",
      "pf",
      "pfizer",
      "pg",
      "ph",
      "pharmacy",
      "phd",
      "philips",
      "phone",
      "photo",
      "photography",
      "photos",
      "physio",
      "piaget",
      "pics",
      "pictet",
      "pictures",
      "pid",
      "pin",
      "ping",
      "pink",
      "pioneer",
      "pizza",
      "pk",
      "pl",
      "place",
      "play",
      "playstation",
      "plumbing",
      "plus",
      "pm",
      "pn",
      "pnc",
      "pohl",
      "poker",
      "politie",
      "porn",
      "post",
      "pr",
      "pramerica",
      "praxi",
      "press",
      "prime",
      "pro",
      "prod",
      "productions",
      "prof",
      "progressive",
      "promo",
      "properties",
      "property",
      "protection",
      "pru",
      "prudential",
      "ps",
      "pt",
      "pub",
      "pw",
      "pwc",
      "py",
      "qa",
      "qpon",
      "quebec",
      "quest",
      "qvc",
      "racing",
      "radio",
      "raid",
      "re",
      "read",
      "realestate",
      "realtor",
      "realty",
      "recipes",
      "red",
      "redstone",
      "redumbrella",
      "rehab",
      "reise",
      "reisen",
      "reit",
      "reliance",
      "ren",
      "rent",
      "rentals",
      "repair",
      "report",
      "republican",
      "rest",
      "restaurant",
      "review",
      "reviews",
      "rexroth",
      "rich",
      "richardli",
      "ricoh",
      "rightathome",
      "ril",
      "rio",
      "rip",
      "rmit",
      "ro",
      "rocher",
      "rocks",
      "rodeo",
      "rogers",
      "room",
      "rs",
      "rsvp",
      "ru",
      "rugby",
      "ruhr",
      "run",
      "rw",
      "rwe",
      "ryukyu",
      "sa",
      "saarland",
      "safe",
      "safety",
      "sakura",
      "sale",
      "salon",
      "samsclub",
      "samsung",
      "sandvik",
      "sandvikcoromant",
      "sanofi",
      "sap",
      "sarl",
      "sas",
      "save",
      "saxo",
      "sb",
      "sbi",
      "sbs",
      "sc",
      "sca",
      "scb",
      "schaeffler",
      "schmidt",
      "scholarships",
      "school",
      "schule",
      "schwarz",
      "science",
      "scjohnson",
      "scor",
      "scot",
      "sd",
      "se",
      "search",
      "seat",
      "secure",
      "security",
      "seek",
      "select",
      "sener",
      "services",
      "ses",
      "seven",
      "sew",
      "sex",
      "sexy",
      "sfr",
      "sg",
      "sh",
      "shangrila",
      "sharp",
      "shaw",
      "shell",
      "shia",
      "shiksha",
      "shoes",
      "shop",
      "shopping",
      "shouji",
      "show",
      "showtime",
      "shriram",
      "si",
      "silk",
      "sina",
      "singles",
      "site",
      "sj",
      "sk",
      "ski",
      "skin",
      "sky",
      "skype",
      "sl",
      "sling",
      "sm",
      "smart",
      "smile",
      "sn",
      "sncf",
      "so",
      "soccer",
      "social",
      "softbank",
      "software",
      "sohu",
      "solar",
      "solutions",
      "song",
      "sony",
      "soy",
      "space",
      "spiegel",
      "sport",
      "spot",
      "spreadbetting",
      "sr",
      "srl",
      "srt",
      "st",
      "stada",
      "staples",
      "star",
      "starhub",
      "statebank",
      "statefarm",
      "statoil",
      "stc",
      "stcgroup",
      "stockholm",
      "storage",
      "store",
      "stream",
      "studio",
      "study",
      "style",
      "su",
      "sucks",
      "supplies",
      "supply",
      "support",
      "surf",
      "surgery",
      "suzuki",
      "sv",
      "swatch",
      "swiftcover",
      "swiss",
      "sx",
      "sy",
      "sydney",
      "symantec",
      "systems",
      "sz",
      "tab",
      "taipei",
      "talk",
      "taobao",
      "target",
      "tatamotors",
      "tatar",
      "tattoo",
      "tax",
      "taxi",
      "tc",
      "tci",
      "td",
      "tdk",
      "team",
      "tech",
      "technology",
      "tel",
      "telecity",
      "telefonica",
      "temasek",
      "tennis",
      "teva",
      "tf",
      "tg",
      "th",
      "thd",
      "theater",
      "theatre",
      "tiaa",
      "tickets",
      "tienda",
      "tiffany",
      "tips",
      "tires",
      "tirol",
      "tj",
      "tjmaxx",
      "tjx",
      "tk",
      "tkmaxx",
      "tl",
      "tm",
      "tmall",
      "tn",
      "to",
      "today",
      "tokyo",
      "tools",
      "top",
      "toray",
      "toshiba",
      "total",
      "tours",
      "town",
      "toyota",
      "toys",
      "tr",
      "trade",
      "trading",
      "training",
      "travel",
      "travelchannel",
      "travelers",
      "travelersinsurance",
      "trust",
      "trv",
      "tt",
      "tube",
      "tui",
      "tunes",
      "tushu",
      "tv",
      "tvs",
      "tw",
      "tz",
      "ua",
      "ubank",
      "ubs",
      "uconnect",
      "ug",
      "uk",
      "unicom",
      "university",
      "uno",
      "uol",
      "ups",
      "us",
      "uy",
      "uz",
      "va",
      "vacations",
      "vana",
      "vanguard",
      "vc",
      "ve",
      "vegas",
      "ventures",
      "verisign",
      "versicherung",
      "vet",
      "vg",
      "vi",
      "viajes",
      "video",
      "vig",
      "viking",
      "villas",
      "vin",
      "vip",
      "virgin",
      "visa",
      "vision",
      "vista",
      "vistaprint",
      "viva",
      "vivo",
      "vlaanderen",
      "vn",
      "vodka",
      "volkswagen",
      "volvo",
      "vote",
      "voting",
      "voto",
      "voyage",
      "vu",
      "vuelos",
      "wales",
      "walmart",
      "walter",
      "wang",
      "wanggou",
      "warman",
      "watch",
      "watches",
      "weather",
      "weatherchannel",
      "webcam",
      "weber",
      "website",
      "wed",
      "wedding",
      "weibo",
      "weir",
      "wf",
      "whoswho",
      "wien",
      "wiki",
      "williamhill",
      "win",
      "windows",
      "wine",
      "winners",
      "wme",
      "wolterskluwer",
      "woodside",
      "work",
      "works",
      "world",
      "wow",
      "ws",
      "wtc",
      "wtf",
      "xbox",
      "xerox",
      "xfinity",
      "xihuan",
      "xin",
      "xn--11b4c3d",
      "xn--1ck2e1b",
      "xn--1qqw23a",
      "xn--2scrj9c",
      "xn--30rr7y",
      "xn--3bst00m",
      "xn--3ds443g",
      "xn--3e0b707e",
      "xn--3hcrj9c",
      "xn--3oq18vl8pn36a",
      "xn--3pxu8k",
      "xn--42c2d9a",
      "xn--45br5cyl",
      "xn--45brj9c",
      "xn--45q11c",
      "xn--4gbrim",
      "xn--54b7fta0cc",
      "xn--55qw42g",
      "xn--55qx5d",
      "xn--5su34j936bgsg",
      "xn--5tzm5g",
      "xn--6frz82g",
      "xn--6qq986b3xl",
      "xn--80adxhks",
      "xn--80ao21a",
      "xn--80aqecdr1a",
      "xn--80asehdb",
      "xn--80aswg",
      "xn--8y0a063a",
      "xn--90a3ac",
      "xn--90ae",
      "xn--90ais",
      "xn--9dbq2a",
      "xn--9et52u",
      "xn--9krt00a",
      "xn--b4w605ferd",
      "xn--bck1b9a5dre4c",
      "xn--c1avg",
      "xn--c2br7g",
      "xn--cck2b3b",
      "xn--cg4bki",
      "xn--clchc0ea0b2g2a9gcd",
      "xn--czr694b",
      "xn--czrs0t",
      "xn--czru2d",
      "xn--d1acj3b",
      "xn--d1alf",
      "xn--e1a4c",
      "xn--eckvdtc9d",
      "xn--efvy88h",
      "xn--estv75g",
      "xn--fct429k",
      "xn--fhbei",
      "xn--fiq228c5hs",
      "xn--fiq64b",
      "xn--fiqs8s",
      "xn--fiqz9s",
      "xn--fjq720a",
      "xn--flw351e",
      "xn--fpcrj9c3d",
      "xn--fzc2c9e2c",
      "xn--fzys8d69uvgm",
      "xn--g2xx48c",
      "xn--gckr3f0f",
      "xn--gecrj9c",
      "xn--gk3at1e",
      "xn--h2breg3eve",
      "xn--h2brj9c",
      "xn--h2brj9c8c",
      "xn--hxt814e",
      "xn--i1b6b1a6a2e",
      "xn--imr513n",
      "xn--io0a7i",
      "xn--j1aef",
      "xn--j1amh",
      "xn--j6w193g",
      "xn--jlq61u9w7b",
      "xn--jvr189m",
      "xn--kcrx77d1x4a",
      "xn--kprw13d",
      "xn--kpry57d",
      "xn--kpu716f",
      "xn--kput3i",
      "xn--l1acc",
      "xn--lgbbat1ad8j",
      "xn--mgb9awbf",
      "xn--mgba3a3ejt",
      "xn--mgba3a4f16a",
      "xn--mgba7c0bbn0a",
      "xn--mgbaakc7dvf",
      "xn--mgbaam7a8h",
      "xn--mgbab2bd",
      "xn--mgbai9azgqp6j",
      "xn--mgbayh7gpa",
      "xn--mgbb9fbpob",
      "xn--mgbbh1a",
      "xn--mgbbh1a71e",
      "xn--mgbc0a9azcg",
      "xn--mgbca7dzdo",
      "xn--mgberp4a5d4ar",
      "xn--mgbgu82a",
      "xn--mgbi4ecexp",
      "xn--mgbpl2fh",
      "xn--mgbt3dhd",
      "xn--mgbtx2b",
      "xn--mgbx4cd0ab",
      "xn--mix891f",
      "xn--mk1bu44c",
      "xn--mxtq1m",
      "xn--ngbc5azd",
      "xn--ngbe9e0a",
      "xn--ngbrx",
      "xn--node",
      "xn--nqv7f",
      "xn--nqv7fs00ema",
      "xn--nyqy26a",
      "xn--o3cw4h",
      "xn--ogbpf8fl",
      "xn--otu796d",
      "xn--p1acf",
      "xn--p1ai",
      "xn--pbt977c",
      "xn--pgbs0dh",
      "xn--pssy2u",
      "xn--q9jyb4c",
      "xn--qcka1pmc",
      "xn--qxam",
      "xn--rhqv96g",
      "xn--rovu88b",
      "xn--rvc1e0am3e",
      "xn--s9brj9c",
      "xn--ses554g",
      "xn--t60b56a",
      "xn--tckwe",
      "xn--tiq49xqyj",
      "xn--unup4y",
      "xn--vermgensberater-ctb",
      "xn--vermgensberatung-pwb",
      "xn--vhquv",
      "xn--vuq861b",
      "xn--w4r85el8fhu5dnra",
      "xn--w4rs40l",
      "xn--wgbh1c",
      "xn--wgbl6a",
      "xn--xhq521b",
      "xn--xkc2al3hye2a",
      "xn--xkc2dl3a5ee0h",
      "xn--y9a3aq",
      "xn--yfro4i67o",
      "xn--ygbi2ammx",
      "xn--zfr164b",
      "xperia",
      "xxx",
      "xyz",
      "yachts",
      "yahoo",
      "yamaxun",
      "yandex",
      "ye",
      "yodobashi",
      "yoga",
      "yokohama",
      "you",
      "youtube",
      "yt",
      "yun",
      "za",
      "zappos",
      "zara",
      "zero",
      "zip",
      "zippo",
      "zm",
      "zone",
      "zuerich",
      "zw"
    };

    Analyzer analyzer =
        new Analyzer() {
          @Override
          protected TokenStreamComponents createComponents(String fieldName) {
            return new TokenStreamComponents(new UAX29URLEmailTokenizer(newAttributeFactory()));
          }
        };

    for (String tld : TLDs) {
      String URL = "example." + tld;
      BaseTokenStreamTestCase.assertAnalyzesTo(
          analyzer, URL, new String[] {URL}, new String[] {"<URL>"});
    }
  }
}