/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.analysis.hunspell;

import static org.apache.lucene.analysis.hunspell.AffixKind.*;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.OfflineSorter;
import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.IntSequenceOutputs;
import org.apache.lucene.util.fst.Util;

/** In-memory structure for the dictionary (.dic) and affix (.aff) data of a hunspell dictionary. */
public class Dictionary {
  // Derived from woorm/LibreOffice dictionaries.
  // See TestAllDictionaries.testMaxPrologueNeeded.
  static final int MAX_PROLOGUE_SCAN_WINDOW = 30 * 1024;

  static final char[] NOFLAGS = new char[0];

  static final char FLAG_UNSET = (char) 0;
  private static final int DEFAULT_FLAGS = 65510;
  static final char HIDDEN_FLAG = (char) 65511; // called 'ONLYUPCASEFLAG' in Hunspell

  static final Charset DEFAULT_CHARSET = StandardCharsets.ISO_8859_1;
  CharsetDecoder decoder = replacingDecoder(DEFAULT_CHARSET);

  FST<IntsRef> prefixes;
  FST<IntsRef> suffixes;
  Breaks breaks = Breaks.DEFAULT;

  /**
   * All condition checks used by prefixes and suffixes. these are typically re-used across many
   * affix stripping rules. so these are deduplicated, to save RAM.
   */
  ArrayList<AffixCondition> patterns = new ArrayList<>();

  /**
   * The entries in the .dic file, mapping to their set of flags. the fst output is the ordinal list
   * for flagLookup.
   */
  FST<IntsRef> words;

  /** A Bloom filter over {@link #words} to avoid unnecessary expensive FST traversals */
  FixedBitSet wordHashes;

  /**
   * The list of unique flagsets (wordforms). theoretically huge, but practically small (for Polish
   * this is 756), otherwise humans wouldn't be able to deal with it either.
   */
  final FlagEnumerator.Lookup flagLookup;

  // the list of unique strip affixes.
  char[] stripData;
  int[] stripOffsets;

  String wordChars = "";

  // 4 chars per affix, each char representing an unsigned 2-byte integer
  char[] affixData = new char[32];
  private int currentAffix = 0;

  // offsets in affixData
  static final int AFFIX_FLAG = 0;
  static final int AFFIX_STRIP_ORD = 1;
  private static final int AFFIX_CONDITION = 2;
  static final int AFFIX_APPEND = 3;

  // Default flag parsing strategy
  FlagParsingStrategy flagParsingStrategy = new SimpleFlagParsingStrategy();

  // AF entries
  private String[] aliases;
  private int aliasCount = 0;

  // AM entries
  private String[] morphAliases;
  private int morphAliasCount = 0;

  final List<String> morphData = new ArrayList<>(Collections.singletonList("")); // empty data at 0

  /**
   * we set this during sorting, so we know to add an extra int (index in {@link #morphData}) to FST
   * output
   */
  boolean hasCustomMorphData;

  boolean ignoreCase;
  boolean checkSharpS;
  boolean complexPrefixes;

  /**
   * All flags used in affix continuation classes. If an outer affix's flag isn't here, there's no
   * need to do 2-level affix stripping with it.
   */
  private char[] secondStagePrefixFlags, secondStageSuffixFlags;

  char circumfix;
  char keepcase, forceUCase;
  char needaffix;
  char forbiddenword;
  char onlyincompound, compoundBegin, compoundMiddle, compoundEnd, compoundFlag;
  char compoundPermit, compoundForbid;
  boolean checkCompoundCase, checkCompoundDup, checkCompoundRep;
  boolean checkCompoundTriple, simplifiedTriple;
  int compoundMin = 3, compoundMax = Integer.MAX_VALUE;
  List<CompoundRule> compoundRules; // nullable
  List<CheckCompoundPattern> checkCompoundPatterns = new ArrayList<>();

  // ignored characters (dictionary, affix, inputs)
  private char[] ignore;

  String tryChars = "";
  String[] neighborKeyGroups = {"qwertyuiop", "asdfghjkl", "zxcvbnm"};
  boolean enableSplitSuggestions = true;
  List<RepEntry> repTable = new ArrayList<>();
  List<List<String>> mapTable = new ArrayList<>();
  int maxDiff = 5;
  int maxNGramSuggestions = 4;
  boolean onlyMaxDiff;
  char noSuggest, subStandard;
  ConvTable iconv, oconv;

  // true if we can strip suffixes "down to nothing"
  boolean fullStrip;

  // language declaration of the dictionary
  String language;
  // true if case algorithms should use alternate (Turkish/Azeri) mapping
  private boolean alternateCasing;

  /**
   * Creates a new Dictionary containing the information read from the provided InputStreams to
   * hunspell affix and dictionary files. You have to close the provided InputStreams yourself.
   *
   * @param tempDir Directory to use for offline sorting
   * @param tempFileNamePrefix prefix to use to generate temp file names
   * @param affix InputStream for reading the hunspell affix file (won't be closed).
   * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
   * @throws IOException Can be thrown while reading from the InputStreams
   * @throws ParseException Can be thrown if the content of the files does not meet expected formats
   */
  public Dictionary(
      Directory tempDir, String tempFileNamePrefix, InputStream affix, InputStream dictionary)
      throws IOException, ParseException {
    this(tempDir, tempFileNamePrefix, affix, Collections.singletonList(dictionary), false);
  }

  /**
   * Creates a new Dictionary containing the information read from the provided InputStreams to
   * hunspell affix and dictionary files. You have to close the provided InputStreams yourself.
   *
   * @param tempDir Directory to use for offline sorting
   * @param tempFileNamePrefix prefix to use to generate temp file names
   * @param affix InputStream for reading the hunspell affix file (won't be closed).
   * @param dictionaries InputStream for reading the hunspell dictionary files (won't be closed).
   * @throws IOException Can be thrown while reading from the InputStreams
   * @throws ParseException Can be thrown if the content of the files does not meet expected formats
   */
  public Dictionary(
      Directory tempDir,
      String tempFileNamePrefix,
      InputStream affix,
      List<InputStream> dictionaries,
      boolean ignoreCase)
      throws IOException, ParseException {
    this.ignoreCase = ignoreCase;

    try (BufferedInputStream affixStream =
        new BufferedInputStream(affix, MAX_PROLOGUE_SCAN_WINDOW) {
          @Override
          public void close() {
            // TODO: maybe we should consume and close it? Why does it need to stay open?
            // Don't close the affix stream as per javadoc.
          }
        }) {
      // I assume we don't support other BOMs (utf16, etc.)? We trivially could,
      // by adding maybeConsume() with a proper bom... but I don't see hunspell repo to have
      // any such exotic examples.
      Charset streamCharset;
      if (maybeConsume(affixStream, BOM_UTF8)) {
        streamCharset = StandardCharsets.UTF_8;
      } else {
        streamCharset = DEFAULT_CHARSET;
      }

      /*
       * pass 1: look for encoding & flag. This is simple but works. We just prefetch
       * a large enough chunk of the input and scan through it. The buffered data will
       * be subsequently reused anyway so nothing is wasted.
       */
      affixStream.mark(MAX_PROLOGUE_SCAN_WINDOW);
      byte[] prologue = affixStream.readNBytes(MAX_PROLOGUE_SCAN_WINDOW - 1);
      affixStream.reset();
      readConfig(new ByteArrayInputStream(prologue), streamCharset);

      // pass 2: parse affixes
      FlagEnumerator flagEnumerator = new FlagEnumerator();
      readAffixFile(affixStream, decoder, flagEnumerator);

      // read dictionary entries
      IndexOutput unsorted = tempDir.createTempOutput(tempFileNamePrefix, "dat", IOContext.DEFAULT);
      int wordCount = mergeDictionaries(dictionaries, decoder, unsorted);
      wordHashes = new FixedBitSet(Integer.highestOneBit(wordCount * 10));
      String sortedFile = sortWordsOffline(tempDir, tempFileNamePrefix, unsorted);
      words = readSortedDictionaries(tempDir, sortedFile, flagEnumerator);
      flagLookup = flagEnumerator.finish();
      aliases = null; // no longer needed
      morphAliases = null; // no longer needed
    }
  }

  int formStep() {
    return hasCustomMorphData ? 2 : 1;
  }

  /** Looks up Hunspell word forms from the dictionary */
  IntsRef lookupWord(char[] word, int offset, int length) {
    int hash = CharsRef.stringHashCode(word, offset, length);
    if (!wordHashes.get(Math.abs(hash) % wordHashes.length())) {
      return null;
    }

    return lookup(words, word, offset, length);
  }

  // only for testing
  IntsRef lookupPrefix(char[] word) {
    return lookup(prefixes, word, 0, word.length);
  }

  // only for testing
  IntsRef lookupSuffix(char[] word) {
    return lookup(suffixes, word, 0, word.length);
  }

  IntsRef lookup(FST<IntsRef> fst, char[] word, int offset, int length) {
    if (fst == null) {
      return null;
    }
    final FST.BytesReader bytesReader = fst.getBytesReader();
    final FST.Arc<IntsRef> arc = fst.getFirstArc(new FST.Arc<>());
    // Accumulate output as we go
    IntsRef output = fst.outputs.getNoOutput();

    int l = offset + length;
    for (int i = offset, cp; i < l; i += Character.charCount(cp)) {
      cp = Character.codePointAt(word, i, l);
      output = nextArc(fst, arc, bytesReader, output, cp);
      if (output == null) {
        return null;
      }
    }
    return nextArc(fst, arc, bytesReader, output, FST.END_LABEL);
  }

  static IntsRef nextArc(
      FST<IntsRef> fst, FST.Arc<IntsRef> arc, FST.BytesReader reader, IntsRef output, int ch) {
    try {
      if (fst.findTargetArc(ch, arc, arc, reader) == null) {
        return null;
      }
    } catch (IOException bogus) {
      throw new RuntimeException(bogus);
    }
    return fst.outputs.add(output, arc.output());
  }

  /**
   * Reads the affix file through the provided InputStream, building up the prefix and suffix maps
   *
   * @param affixStream InputStream to read the content of the affix file from
   * @param decoder CharsetDecoder to decode the content of the file
   * @throws IOException Can be thrown while reading from the InputStream
   */
  private void readAffixFile(InputStream affixStream, CharsetDecoder decoder, FlagEnumerator flags)
      throws IOException, ParseException {
    TreeMap<String, List<Integer>> prefixes = new TreeMap<>();
    TreeMap<String, List<Integer>> suffixes = new TreeMap<>();
    Set<Character> prefixContFlags = new HashSet<>();
    Set<Character> suffixContFlags = new HashSet<>();
    Map<String, Integer> seenPatterns = new HashMap<>();

    // zero condition -> 0 ord
    seenPatterns.put(AffixCondition.ALWAYS_TRUE_KEY, 0);
    patterns.add(null);

    // zero strip -> 0 ord
    Map<String, Integer> seenStrips = new LinkedHashMap<>();
    seenStrips.put("", 0);

    LineNumberReader reader = new LineNumberReader(new InputStreamReader(affixStream, decoder));
    String line;
    while ((line = reader.readLine()) != null) {
      // ignore any BOM marker on first line
      if (reader.getLineNumber() == 1 && line.startsWith("\uFEFF")) {
        line = line.substring(1);
      }
      line = line.trim();
      if (line.isEmpty()) continue;

      String firstWord = line.split("\\s")[0];
      // TODO: convert to a switch?
      if ("AF".equals(firstWord)) {
        parseAlias(line);
      } else if ("AM".equals(firstWord)) {
        parseMorphAlias(line);
      } else if ("PFX".equals(firstWord)) {
        parseAffix(
            prefixes, prefixContFlags, line, reader, PREFIX, seenPatterns, seenStrips, flags);
      } else if ("SFX".equals(firstWord)) {
        parseAffix(
            suffixes, suffixContFlags, line, reader, SUFFIX, seenPatterns, seenStrips, flags);
      } else if (line.equals("COMPLEXPREFIXES")) {
        complexPrefixes =
            true; // 2-stage prefix+1-stage suffix instead of 2-stage suffix+1-stage prefix
      } else if ("CIRCUMFIX".equals(firstWord)) {
        circumfix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("KEEPCASE".equals(firstWord)) {
        keepcase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("FORCEUCASE".equals(firstWord)) {
        forceUCase = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("NEEDAFFIX".equals(firstWord) || "PSEUDOROOT".equals(firstWord)) {
        needaffix = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("ONLYINCOMPOUND".equals(firstWord)) {
        onlyincompound = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("CHECKSHARPS".equals(firstWord)) {
        checkSharpS = true;
      } else if ("IGNORE".equals(firstWord)) {
        ignore = singleArgument(reader, line).toCharArray();
        Arrays.sort(ignore);
      } else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {
        int num = parseNum(reader, line);
        ConvTable res = parseConversions(reader, num);
        if (line.startsWith("I")) {
          iconv = res;
        } else {
          oconv = res;
        }
      } else if ("FULLSTRIP".equals(firstWord)) {
        fullStrip = true;
      } else if ("LANG".equals(firstWord)) {
        language = singleArgument(reader, line);
        this.alternateCasing = hasLanguage("tr", "az");
      } else if ("BREAK".equals(firstWord)) {
        breaks = parseBreaks(reader, line);
      } else if ("WORDCHARS".equals(firstWord)) {
        wordChars = firstArgument(reader, line);
      } else if ("TRY".equals(firstWord)) {
        tryChars = firstArgument(reader, line);
      } else if ("REP".equals(firstWord)) {
        int count = parseNum(reader, line);
        for (int i = 0; i < count; i++) {
          String[] parts = splitBySpace(reader, reader.readLine(), 3, Integer.MAX_VALUE);
          repTable.add(new RepEntry(parts[1], parts[2]));
        }
      } else if ("MAP".equals(firstWord)) {
        int count = parseNum(reader, line);
        for (int i = 0; i < count; i++) {
          mapTable.add(parseMapEntry(reader, reader.readLine()));
        }
      } else if ("KEY".equals(firstWord)) {
        neighborKeyGroups = singleArgument(reader, line).split("\\|");
      } else if ("NOSPLITSUGS".equals(firstWord)) {
        enableSplitSuggestions = false;
      } else if ("MAXNGRAMSUGS".equals(firstWord)) {
        maxNGramSuggestions = Integer.parseInt(singleArgument(reader, line));
      } else if ("MAXDIFF".equals(firstWord)) {
        int i = Integer.parseInt(singleArgument(reader, line));
        if (i < 0 || i > 10) {
          throw new ParseException("MAXDIFF should be between 0 and 10", reader.getLineNumber());
        }
        maxDiff = i;
      } else if ("ONLYMAXDIFF".equals(firstWord)) {
        onlyMaxDiff = true;
      } else if ("FORBIDDENWORD".equals(firstWord)) {
        forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("NOSUGGEST".equals(firstWord)) {
        noSuggest = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("SUBSTANDARD".equals(firstWord)) {
        subStandard = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDMIN".equals(firstWord)) {
        compoundMin = Math.max(1, parseNum(reader, line));
      } else if ("COMPOUNDWORDMAX".equals(firstWord)) {
        compoundMax = Math.max(1, parseNum(reader, line));
      } else if ("COMPOUNDRULE".equals(firstWord)) {
        compoundRules = parseCompoundRules(reader, parseNum(reader, line));
      } else if ("COMPOUNDFLAG".equals(firstWord)) {
        compoundFlag = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDBEGIN".equals(firstWord)) {
        compoundBegin = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDMIDDLE".equals(firstWord)) {
        compoundMiddle = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDEND".equals(firstWord)) {
        compoundEnd = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDPERMITFLAG".equals(firstWord)) {
        compoundPermit = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("COMPOUNDFORBIDFLAG".equals(firstWord)) {
        compoundForbid = flagParsingStrategy.parseFlag(singleArgument(reader, line));
      } else if ("CHECKCOMPOUNDCASE".equals(firstWord)) {
        checkCompoundCase = true;
      } else if ("CHECKCOMPOUNDDUP".equals(firstWord)) {
        checkCompoundDup = true;
      } else if ("CHECKCOMPOUNDREP".equals(firstWord)) {
        checkCompoundRep = true;
      } else if ("CHECKCOMPOUNDTRIPLE".equals(firstWord)) {
        checkCompoundTriple = true;
      } else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {
        simplifiedTriple = true;
      } else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {
        int count = parseNum(reader, line);
        for (int i = 0; i < count; i++) {
          checkCompoundPatterns.add(
              new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));
        }
      } else if ("SET".equals(firstWord)) {
        checkCriticalDirectiveSame(
            "SET", reader, decoder.charset(), getDecoder(singleArgument(reader, line)).charset());
      } else if ("FLAG".equals(firstWord)) {
        FlagParsingStrategy strategy = getFlagParsingStrategy(line, decoder.charset());
        checkCriticalDirectiveSame(
            "FLAG", reader, flagParsingStrategy.getClass(), strategy.getClass());
      }
    }

    this.prefixes = affixFST(prefixes);
    this.suffixes = affixFST(suffixes);
    secondStagePrefixFlags = toSortedCharArray(prefixContFlags);
    secondStageSuffixFlags = toSortedCharArray(suffixContFlags);

    int totalChars = 0;
    for (String strip : seenStrips.keySet()) {
      totalChars += strip.length();
    }
    stripData = new char[totalChars];
    stripOffsets = new int[seenStrips.size() + 1];
    int currentOffset = 0;
    int currentIndex = 0;
    for (String strip : seenStrips.keySet()) {
      stripOffsets[currentIndex++] = currentOffset;
      strip.getChars(0, strip.length(), stripData, currentOffset);
      currentOffset += strip.length();
    }
    assert currentIndex == seenStrips.size();
    stripOffsets[currentIndex] = currentOffset;
  }

  private void checkCriticalDirectiveSame(
      String directive, LineNumberReader reader, Object expected, Object actual)
      throws ParseException {
    if (!expected.equals(actual)) {
      throw new ParseException(
          directive
              + " directive should occur at most once, and in the first "
              + MAX_PROLOGUE_SCAN_WINDOW
              + " bytes of the *.aff file",
          reader.getLineNumber());
    }
  }

  private List<String> parseMapEntry(LineNumberReader reader, String line) throws ParseException {
    String unparsed = firstArgument(reader, line);
    List<String> mapEntry = new ArrayList<>();
    for (int j = 0; j < unparsed.length(); j++) {
      if (unparsed.charAt(j) == '(') {
        int closing = unparsed.indexOf(')', j);
        if (closing < 0) {
          throw new ParseException("Unclosed parenthesis: " + line, reader.getLineNumber());
        }

        mapEntry.add(unparsed.substring(j + 1, closing));
        j = closing;
      } else {
        mapEntry.add(String.valueOf(unparsed.charAt(j)));
      }
    }
    return mapEntry;
  }

  boolean hasLanguage(String... langCodes) {
    if (language == null) return false;
    String langCode = extractLanguageCode(language);
    for (String code : langCodes) {
      if (langCode.equals(code)) {
        return true;
      }
    }
    return false;
  }

  /**
   * @param root a string to look up in the dictionary. No case conversion or affix removal is
   *     performed. To get the possible roots of any word, you may call {@link
   *     Hunspell#getRoots(String)}
   * @return the dictionary entries for the given root, or {@code null} if there's none
   */
  public DictEntries lookupEntries(String root) {
    IntsRef forms = lookupWord(root.toCharArray(), 0, root.length());
    if (forms == null) return null;

    return new DictEntries() {
      @Override
      public int size() {
        return forms.length / (hasCustomMorphData ? 2 : 1);
      }

      @Override
      public String getMorphologicalData(int entryIndex) {
        if (!hasCustomMorphData) return "";
        return morphData.get(forms.ints[forms.offset + entryIndex * 2 + 1]);
      }

      @Override
      public List<String> getMorphologicalValues(int entryIndex, String key) {
        assert key.length() == 3;
        assert key.charAt(2) == ':';

        String fields = getMorphologicalData(entryIndex);
        if (fields.isEmpty() || !fields.contains(key)) return Collections.emptyList();

        return Arrays.stream(fields.split(" "))
            .filter(s -> s.startsWith(key))
            .map(s -> s.substring(3))
            .collect(Collectors.toList());
      }
    };
  }

  static String extractLanguageCode(String isoCode) {
    int underscore = isoCode.indexOf("_");
    return underscore < 0 ? isoCode : isoCode.substring(0, underscore);
  }

  private int parseNum(LineNumberReader reader, String line) throws ParseException {
    return Integer.parseInt(splitBySpace(reader, line, 2, Integer.MAX_VALUE)[1]);
  }

  private String singleArgument(LineNumberReader reader, String line) throws ParseException {
    return splitBySpace(reader, line, 2)[1];
  }

  private String firstArgument(LineNumberReader reader, String line) throws ParseException {
    return splitBySpace(reader, line, 2, Integer.MAX_VALUE)[1];
  }

  private String[] splitBySpace(LineNumberReader reader, String line, int expectedParts)
      throws ParseException {
    return splitBySpace(reader, line, expectedParts, expectedParts);
  }

  private String[] splitBySpace(LineNumberReader reader, String line, int minParts, int maxParts)
      throws ParseException {
    String[] parts = line.split("\\s+");
    if (parts.length < minParts || parts.length > maxParts && !parts[maxParts].startsWith("#")) {
      throw new ParseException("Invalid syntax: " + line, reader.getLineNumber());
    }
    return parts;
  }

  private List<CompoundRule> parseCompoundRules(LineNumberReader reader, int num)
      throws IOException, ParseException {
    List<CompoundRule> compoundRules = new ArrayList<>();
    for (int i = 0; i < num; i++) {
      compoundRules.add(new CompoundRule(singleArgument(reader, reader.readLine()), this));
    }
    return compoundRules;
  }

  private Breaks parseBreaks(LineNumberReader reader, String line)
      throws IOException, ParseException {
    Set<String> starting = new LinkedHashSet<>();
    Set<String> ending = new LinkedHashSet<>();
    Set<String> middle = new LinkedHashSet<>();
    int num = parseNum(reader, line);
    for (int i = 0; i < num; i++) {
      String breakStr = singleArgument(reader, reader.readLine());
      if (breakStr.startsWith("^")) {
        starting.add(breakStr.substring(1));
      } else if (breakStr.endsWith("$")) {
        ending.add(breakStr.substring(0, breakStr.length() - 1));
      } else {
        middle.add(breakStr);
      }
    }
    return new Breaks(starting, ending, middle);
  }

  private FST<IntsRef> affixFST(TreeMap<String, List<Integer>> affixes) throws IOException {
    IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton();
    FSTCompiler<IntsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, outputs);
    IntsRefBuilder scratch = new IntsRefBuilder();
    for (Map.Entry<String, List<Integer>> entry : affixes.entrySet()) {
      Util.toUTF32(entry.getKey(), scratch);
      List<Integer> entries = entry.getValue();
      IntsRef output = new IntsRef(entries.size());
      for (Integer c : entries) {
        output.ints[output.length++] = c;
      }
      fstCompiler.add(scratch.get(), output);
    }
    return fstCompiler.compile();
  }

  /**
   * Parses a specific affix rule putting the result into the provided affix map
   *
   * @param affixes Map where the result of the parsing will be put
   * @param header Header line of the affix rule
   * @param reader BufferedReader to read the content of the rule from
   * @param seenPatterns map from condition -&gt; index of patterns, for deduplication.
   * @throws IOException Can be thrown while reading the rule
   */
  private void parseAffix(
      TreeMap<String, List<Integer>> affixes,
      Set<Character> secondStageFlags,
      String header,
      LineNumberReader reader,
      AffixKind kind,
      Map<String, Integer> seenPatterns,
      Map<String, Integer> seenStrips,
      FlagEnumerator flags)
      throws IOException, ParseException {

    StringBuilder sb = new StringBuilder();
    String[] args = header.split("\\s+");

    boolean crossProduct = args[2].equals("Y");

    int numLines;
    try {
      numLines = Integer.parseInt(args[3]);
    } catch (NumberFormatException e) {
      return;
    }
    affixData = ArrayUtil.grow(affixData, currentAffix * 4 + numLines * 4);

    for (int i = 0; i < numLines; i++) {
      String line = reader.readLine();
      // from the manpage: PFX flag stripping prefix [condition [morphological_fields...]]
      String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);

      char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);
      String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];
      String affixArg = ruleArgs[3];
      char[] appendFlags = null;

      // first: parse continuation classes out of affix
      int flagSep = affixArg.lastIndexOf('/');
      if (flagSep != -1) {
        String flagPart = affixArg.substring(flagSep + 1);
        affixArg = affixArg.substring(0, flagSep);

        if (aliasCount > 0) {
          flagPart = getAliasValue(Integer.parseInt(flagPart));
        }

        appendFlags = flagParsingStrategy.parseFlags(flagPart);
        for (char appendFlag : appendFlags) {
          secondStageFlags.add(appendFlag);
        }
      }
      // zero affix -> empty string
      if ("0".equals(affixArg)) {
        affixArg = "";
      }

      String condition = ruleArgs.length > 4 ? ruleArgs[4] : ".";
      String key = AffixCondition.uniqueKey(kind, strip, condition);

      // deduplicate patterns
      Integer patternIndex = seenPatterns.get(key);
      if (patternIndex == null) {
        patternIndex = patterns.size();
        if (patternIndex > Short.MAX_VALUE) {
          throw new UnsupportedOperationException(
              "Too many patterns, please report this to dev@lucene.apache.org");
        }
        seenPatterns.put(key, patternIndex);
        patterns.add(AffixCondition.compile(kind, strip, condition, line));
      }

      Integer stripOrd = seenStrips.get(strip);
      if (stripOrd == null) {
        stripOrd = seenStrips.size();
        seenStrips.put(strip, stripOrd);
        if (stripOrd > Character.MAX_VALUE) {
          throw new UnsupportedOperationException(
              "Too many unique strips, please report this to dev@lucene.apache.org");
        }
      }

      if (appendFlags == null) {
        appendFlags = NOFLAGS;
      }

      int appendFlagsOrd = flags.add(appendFlags);
      if (appendFlagsOrd < 0) {
        // already exists in our hash
        appendFlagsOrd = (-appendFlagsOrd) - 1;
      } else if (appendFlagsOrd > Short.MAX_VALUE) {
        // this limit is probably flexible, but it's a good sanity check too
        throw new UnsupportedOperationException(
            "Too many unique append flags, please report this to dev@lucene.apache.org");
      }

      int dataStart = currentAffix * 4;
      affixData[dataStart + AFFIX_FLAG] = flag;
      affixData[dataStart + AFFIX_STRIP_ORD] = (char) stripOrd.intValue();
      // encode crossProduct into patternIndex
      int patternOrd = patternIndex << 1 | (crossProduct ? 1 : 0);
      affixData[dataStart + AFFIX_CONDITION] = (char) patternOrd;
      affixData[dataStart + AFFIX_APPEND] = (char) appendFlagsOrd;

      if (needsInputCleaning(affixArg)) {
        affixArg = cleanInput(affixArg, sb).toString();
      }

      if (kind == SUFFIX) {
        affixArg = new StringBuilder(affixArg).reverse().toString();
      }

      affixes.computeIfAbsent(affixArg, __ -> new ArrayList<>()).add(currentAffix);
      currentAffix++;
    }
  }

  char affixData(int affixIndex, int offset) {
    return affixData[affixIndex * 4 + offset];
  }

  boolean isCrossProduct(int affix) {
    return (affixData(affix, AFFIX_CONDITION) & 1) == 1;
  }

  int getAffixCondition(int affix) {
    return affixData(affix, AFFIX_CONDITION) >>> 1;
  }

  private ConvTable parseConversions(LineNumberReader reader, int num)
      throws IOException, ParseException {
    TreeMap<String, String> mappings = new TreeMap<>();

    for (int i = 0; i < num; i++) {
      String[] parts = splitBySpace(reader, reader.readLine(), 3);
      if (mappings.put(parts[1], parts[2]) != null) {
        throw new IllegalStateException("duplicate mapping specified for: " + parts[1]);
      }
    }

    return new ConvTable(mappings);
  }

  private static final byte[] BOM_UTF8 = {(byte) 0xef, (byte) 0xbb, (byte) 0xbf};

  /** Parses the encoding and flag format specified in the provided InputStream */
  private void readConfig(InputStream stream, Charset streamCharset)
      throws IOException, ParseException {
    LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, streamCharset));
    String line;
    String flagLine = null;
    boolean charsetFound = false;
    boolean flagFound = false;
    while ((line = reader.readLine()) != null) {
      if (line.isBlank()) continue;

      String firstWord = line.split("\\s")[0];
      if ("SET".equals(firstWord)) {
        decoder = getDecoder(singleArgument(reader, line));
        charsetFound = true;
      } else if ("FLAG".equals(firstWord)) {
        // Preserve the flag line for parsing later since we need the decoder's charset
        // and just in case they come out of order.
        flagLine = line;
        flagFound = true;
      } else {
        continue;
      }

      if (charsetFound && flagFound) {
        break;
      }
    }

    if (flagFound) {
      flagParsingStrategy = getFlagParsingStrategy(flagLine, decoder.charset());
    }
  }

  /**
   * Consume the provided byte sequence in full, if present. Otherwise leave the input stream
   * intact.
   *
   * @return {@code true} if the sequence matched and has been consumed.
   */
  @SuppressWarnings("SameParameterValue")
  private static boolean maybeConsume(BufferedInputStream stream, byte[] bytes) throws IOException {
    stream.mark(bytes.length);
    for (byte b : bytes) {
      int nextByte = stream.read();
      if (nextByte != (b & 0xff)) { // covers EOF (-1) as well.
        stream.reset();
        return false;
      }
    }
    return true;
  }

  static final Map<String, String> CHARSET_ALIASES =
      Map.of("microsoft-cp1251", "windows-1251", "TIS620-2533", "TIS-620");

  /**
   * Retrieves the CharsetDecoder for the given encoding. Note, This isn't perfect as I think
   * ISCII-DEVANAGARI and MICROSOFT-CP1251 etc are allowed...
   *
   * @param encoding Encoding to retrieve the CharsetDecoder for
   * @return CharSetDecoder for the given encoding
   */
  private CharsetDecoder getDecoder(String encoding) {
    if ("ISO8859-14".equals(encoding)) {
      return new ISO8859_14Decoder();
    }
    String canon = CHARSET_ALIASES.get(encoding);
    if (canon != null) {
      encoding = canon;
    }
    return replacingDecoder(Charset.forName(encoding));
  }

  private static CharsetDecoder replacingDecoder(Charset charset) {
    return charset.newDecoder().onMalformedInput(CodingErrorAction.REPLACE);
  }

  /**
   * Determines the appropriate {@link FlagParsingStrategy} based on the FLAG definition line taken
   * from the affix file
   *
   * @param flagLine Line containing the flag information
   * @return FlagParsingStrategy that handles parsing flags in the way specified in the FLAG
   *     definition
   */
  static FlagParsingStrategy getFlagParsingStrategy(String flagLine, Charset charset) {
    String[] parts = flagLine.split("\\s+");
    if (parts.length != 2) {
      throw new IllegalArgumentException("Illegal FLAG specification: " + flagLine);
    }
    String flagType = parts[1];

    if ("num".equals(flagType)) {
      return new NumFlagParsingStrategy();
    } else if ("UTF-8".equals(flagType)) {
      if (DEFAULT_CHARSET.equals(charset)) {
        return new DefaultAsUtf8FlagParsingStrategy();
      }
      return new SimpleFlagParsingStrategy();
    } else if ("long".equals(flagType)) {
      return new DoubleASCIIFlagParsingStrategy();
    }

    throw new IllegalArgumentException("Unknown flag type: " + flagType);
  }

  private static final char FLAG_SEPARATOR = 0x1f; // flag separator after escaping
  private static final char MORPH_SEPARATOR =
      0x1e; // separator for boundary of entry (may be followed by morph data)

  private String unescapeEntry(String entry) {
    StringBuilder sb = new StringBuilder();
    int end = morphBoundary(entry);
    for (int i = 0; i < end; i++) {
      char ch = entry.charAt(i);
      if (ch == '\\' && i + 1 < entry.length()) {
        sb.append(entry.charAt(i + 1));
        i++;
      } else if (ch == '/' && i > 0) {
        sb.append(FLAG_SEPARATOR);
      } else if (!shouldSkipEscapedChar(ch)) {
        sb.append(ch);
      }
    }
    sb.append(MORPH_SEPARATOR);
    if (end < entry.length()) {
      for (int i = end; i < entry.length(); i++) {
        char c = entry.charAt(i);
        if (!shouldSkipEscapedChar(c)) {
          sb.append(c);
        }
      }
    }
    return sb.toString();
  }

  private static boolean shouldSkipEscapedChar(char ch) {
    return ch == FLAG_SEPARATOR
        || ch == MORPH_SEPARATOR; // BINARY EXECUTABLES EMBEDDED IN ZULU DICTIONARIES!!!!!!!
  }

  private static int morphBoundary(String line) {
    int end = indexOfSpaceOrTab(line, 0);
    if (end == -1) {
      return line.length();
    }
    while (end >= 0 && end < line.length()) {
      if (line.charAt(end) == '\t'
          || end > 0
              && end + 3 < line.length()
              && Character.isLetter(line.charAt(end + 1))
              && Character.isLetter(line.charAt(end + 2))
              && line.charAt(end + 3) == ':') {
        break;
      }
      end = indexOfSpaceOrTab(line, end + 1);
    }
    if (end == -1) {
      return line.length();
    }
    return end;
  }

  static int indexOfSpaceOrTab(String text, int start) {
    int pos1 = text.indexOf('\t', start);
    int pos2 = text.indexOf(' ', start);
    if (pos1 >= 0 && pos2 >= 0) {
      return Math.min(pos1, pos2);
    } else {
      return Math.max(pos1, pos2);
    }
  }

  private int mergeDictionaries(
      List<InputStream> dictionaries, CharsetDecoder decoder, IndexOutput output)
      throws IOException {
    StringBuilder sb = new StringBuilder();
    int wordCount = 0;
    try (ByteSequencesWriter writer = new ByteSequencesWriter(output)) {
      for (InputStream dictionary : dictionaries) {
        BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));
        lines.readLine(); // first line is number of entries (approximately, sometimes)

        String line;
        while ((line = lines.readLine()) != null) {
          // wild and unpredictable code comment rules
          if (line.isEmpty() || line.charAt(0) == '#' || line.charAt(0) == '\t') {
            continue;
          }
          line = unescapeEntry(line);
          // if we haven't seen any custom morphological data, try to parse one
          if (!hasCustomMorphData) {
            int morphStart = line.indexOf(MORPH_SEPARATOR);
            if (morphStart >= 0 && morphStart < line.length()) {
              String data = line.substring(morphStart + 1);
              hasCustomMorphData =
                  splitMorphData(data).stream().anyMatch(s -> !s.startsWith("ph:"));
            }
          }

          wordCount += writeNormalizedWordEntry(sb, writer, line);
        }
      }
      CodecUtil.writeFooter(output);
    }
    return wordCount;
  }

  /** @return the number of word entries written */
  private int writeNormalizedWordEntry(StringBuilder reuse, ByteSequencesWriter writer, String line)
      throws IOException {
    int flagSep = line.indexOf(FLAG_SEPARATOR);
    int morphSep = line.indexOf(MORPH_SEPARATOR);
    assert morphSep > 0;
    assert morphSep > flagSep;
    int sep = flagSep < 0 ? morphSep : flagSep;
    if (sep == 0) return 0;

    CharSequence toWrite;
    String beforeSep = line.substring(0, sep);
    if (needsInputCleaning(beforeSep)) {
      cleanInput(beforeSep, reuse);
      reuse.append(line, sep, line.length());
      toWrite = reuse;
    } else {
      toWrite = line;
    }

    String written = toWrite.toString();
    sep = written.length() - (line.length() - sep);
    writer.write(written.getBytes(StandardCharsets.UTF_8));

    WordCase wordCase = WordCase.caseOf(written, sep);
    if (wordCase == WordCase.MIXED || wordCase == WordCase.UPPER && flagSep > 0) {
      addHiddenCapitalizedWord(reuse, writer, written.substring(0, sep), written.substring(sep));
      return 2;
    }
    return 1;
  }

  private void addHiddenCapitalizedWord(
      StringBuilder reuse, ByteSequencesWriter writer, String word, String afterSep)
      throws IOException {
    reuse.setLength(0);
    reuse.append(Character.toUpperCase(word.charAt(0)));
    for (int i = 1; i < word.length(); i++) {
      reuse.append(caseFold(word.charAt(i)));
    }
    reuse.append(FLAG_SEPARATOR);
    reuse.append(HIDDEN_FLAG);
    reuse.append(afterSep, afterSep.charAt(0) == FLAG_SEPARATOR ? 1 : 0, afterSep.length());
    writer.write(reuse.toString().getBytes(StandardCharsets.UTF_8));
  }

  String toLowerCase(String word) {
    char[] chars = new char[word.length()];
    for (int i = 0; i < word.length(); i++) {
      chars[i] = caseFold(word.charAt(i));
    }
    return new String(chars);
  }

  String toTitleCase(String word) {
    char[] chars = new char[word.length()];
    chars[0] = Character.toUpperCase(word.charAt(0));
    for (int i = 1; i < word.length(); i++) {
      chars[i] = caseFold(word.charAt(i));
    }
    return new String(chars);
  }

  private String sortWordsOffline(
      Directory tempDir, String tempFileNamePrefix, IndexOutput unsorted) throws IOException {
    OfflineSorter sorter =
        new OfflineSorter(
            tempDir,
            tempFileNamePrefix,
            new Comparator<>() {
              final BytesRef scratch1 = new BytesRef();
              final BytesRef scratch2 = new BytesRef();

              private void initScratch(BytesRef o, BytesRef scratch) {
                scratch.bytes = o.bytes;
                scratch.offset = o.offset;
                scratch.length = o.length;

                for (int i = scratch.length - 1; i >= 0; i--) {
                  if (scratch.bytes[scratch.offset + i] == FLAG_SEPARATOR
                      || scratch.bytes[scratch.offset + i] == MORPH_SEPARATOR) {
                    scratch.length = i;
                    break;
                  }
                }
              }

              @Override
              public int compare(BytesRef o1, BytesRef o2) {
                initScratch(o1, scratch1);
                initScratch(o2, scratch2);

                int cmp = scratch1.compareTo(scratch2);
                if (cmp == 0) {
                  // tie break on whole row
                  return o1.compareTo(o2);
                } else {
                  return cmp;
                }
              }
            });

    String sorted;
    boolean success = false;
    try {
      sorted = sorter.sort(unsorted.getName());
      success = true;
    } finally {
      if (success) {
        tempDir.deleteFile(unsorted.getName());
      } else {
        IOUtils.deleteFilesIgnoringExceptions(tempDir, unsorted.getName());
      }
    }
    return sorted;
  }

  private FST<IntsRef> readSortedDictionaries(
      Directory tempDir, String sorted, FlagEnumerator flags) throws IOException {
    boolean success = false;

    Map<String, Integer> morphIndices = new HashMap<>();

    EntryGrouper grouper = new EntryGrouper(flags);

    try (ByteSequencesReader reader =
        new ByteSequencesReader(tempDir.openChecksumInput(sorted, IOContext.READONCE), sorted)) {

      // TODO: the flags themselves can be double-chars (long) or also numeric
      // either way the trick is to encode them as char... but they must be parsed differently

      while (true) {
        BytesRef scratch = reader.next();
        if (scratch == null) {
          break;
        }

        String line = scratch.utf8ToString();
        String entry;
        char[] wordForm;
        int end;

        int flagSep = line.indexOf(FLAG_SEPARATOR);
        if (flagSep == -1) {
          wordForm = NOFLAGS;
          end = line.indexOf(MORPH_SEPARATOR);
          entry = line.substring(0, end);
        } else {
          end = line.indexOf(MORPH_SEPARATOR);
          boolean hidden = line.charAt(flagSep + 1) == HIDDEN_FLAG;
          String flagPart = line.substring(flagSep + (hidden ? 2 : 1), end);
          if (aliasCount > 0 && !flagPart.isEmpty()) {
            flagPart = getAliasValue(Integer.parseInt(flagPart));
          }

          wordForm = flagParsingStrategy.parseFlags(flagPart);
          if (hidden) {
            wordForm = ArrayUtil.growExact(wordForm, wordForm.length + 1);
            wordForm[wordForm.length - 1] = HIDDEN_FLAG;
          }
          entry = line.substring(0, flagSep);
        }

        int morphDataID = 0;
        if (end + 1 < line.length()) {
          List<String> morphFields = readMorphFields(entry, line.substring(end + 1));
          if (!morphFields.isEmpty()) {
            morphFields.sort(Comparator.naturalOrder());
            morphDataID = addMorphFields(morphIndices, String.join(" ", morphFields));
          }
        }

        wordHashes.set(Math.abs(entry.hashCode()) % wordHashes.length());
        grouper.add(entry, wordForm, morphDataID);
      }

      // finalize last entry
      grouper.flushGroup();
      success = true;
      return grouper.words.compile();
    } finally {
      if (success) {
        tempDir.deleteFile(sorted);
      } else {
        IOUtils.deleteFilesIgnoringExceptions(tempDir, sorted);
      }
    }
  }

  private List<String> readMorphFields(String word, String unparsed) {
    List<String> morphFields = null;
    for (String datum : splitMorphData(unparsed)) {
      if (datum.startsWith("ph:")) {
        addPhoneticRepEntries(word, datum.substring(3));
      } else {
        if (morphFields == null) morphFields = new ArrayList<>(1);
        morphFields.add(datum);
      }
    }
    return morphFields == null ? Collections.emptyList() : morphFields;
  }

  private int addMorphFields(Map<String, Integer> indices, String morphFields) {
    Integer alreadyCached = indices.get(morphFields);
    if (alreadyCached != null) {
      return alreadyCached;
    }

    int index = morphData.size();
    indices.put(morphFields, index);
    morphData.add(morphFields);
    return index;
  }

  private void addPhoneticRepEntries(String word, String ph) {
    // e.g. "pretty ph:prity ph:priti->pretti" to suggest both prity->pretty and pritier->prettiest
    int arrow = ph.indexOf("->");
    String pattern;
    String replacement;
    if (arrow > 0) {
      pattern = ph.substring(0, arrow);
      replacement = ph.substring(arrow + 2);
    } else {
      pattern = ph;
      replacement = word;
    }

    // when the ph: field ends with *, strip last character of pattern and replacement
    // e.g., "pretty ph:prity*" results in "prit->prett" replacement instead of "prity->pretty",
    // to get both prity->pretty and pritiest->prettiest suggestions.
    if (pattern.endsWith("*") && pattern.length() > 2 && replacement.length() > 1) {
      pattern = pattern.substring(0, pattern.length() - 2);
      replacement = replacement.substring(0, replacement.length() - 1);
    }

    // capitalize lowercase pattern for capitalized words to support
    // good suggestions also for capitalized misspellings,
    // e.g. Wednesday ph:wendsay results in wendsay -> Wednesday and Wendsay -> Wednesday.
    if (WordCase.caseOf(word) == WordCase.TITLE && WordCase.caseOf(pattern) == WordCase.LOWER) {
      // add also lowercase word in the case of German or
      // Hungarian to support lowercase suggestions lowercased by
      // compound word generation or derivational suffixes
      // for example by adjectival suffix "-i" of geographical names in Hungarian:
      // Massachusetts ph:messzecsuzec
      // messzecsuzeci -> massachusettsi (adjective)
      // For lowercasing by conditional PFX rules, see e.g. germancompounding test
      if (hasLanguage("de", "hu")) {
        repTable.add(new RepEntry(pattern, toLowerCase(replacement)));
      }
      repTable.add(new RepEntry(toTitleCase(pattern), replacement));
    }
    repTable.add(new RepEntry(pattern, replacement));
  }

  boolean isDotICaseChangeDisallowed(char[] word) {
    return word[0] == 'İ' && !alternateCasing;
  }

  private class EntryGrouper {
    final FSTCompiler<IntsRef> words =
        new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, IntSequenceOutputs.getSingleton());
    private final List<char[]> group = new ArrayList<>();
    private final List<Integer> morphDataIDs = new ArrayList<>();
    private final IntsRefBuilder scratchInts = new IntsRefBuilder();
    private String currentEntry = null;
    private final FlagEnumerator flagEnumerator;

    EntryGrouper(FlagEnumerator flagEnumerator) {
      this.flagEnumerator = flagEnumerator;
    }

    void add(String entry, char[] flags, int morphDataID) throws IOException {
      if (!entry.equals(currentEntry)) {
        if (currentEntry != null) {
          if (entry.compareTo(currentEntry) < 0) {
            throw new IllegalArgumentException("out of order: " + entry + " < " + currentEntry);
          }
          flushGroup();
        }
        currentEntry = entry;
      }

      group.add(flags);
      if (hasCustomMorphData) {
        morphDataIDs.add(morphDataID);
      }
    }

    void flushGroup() throws IOException {
      IntsRefBuilder currentOrds = new IntsRefBuilder();

      boolean hasNonHidden = false;
      for (char[] flags : group) {
        if (!hasHiddenFlag(flags)) {
          hasNonHidden = true;
          break;
        }
      }

      for (int i = 0; i < group.size(); i++) {
        char[] flags = group.get(i);
        if (hasNonHidden && hasHiddenFlag(flags)) {
          continue;
        }

        currentOrds.append(flagEnumerator.add(flags));
        if (hasCustomMorphData) {
          currentOrds.append(morphDataIDs.get(i));
        }
      }

      Util.toUTF32(currentEntry, scratchInts);
      words.add(scratchInts.get(), currentOrds.get());

      group.clear();
      morphDataIDs.clear();
    }
  }

  private static boolean hasHiddenFlag(char[] flags) {
    for (char flag : flags) {
      if (flag == HIDDEN_FLAG) {
        return true;
      }
    }
    return false;
  }

  private void parseAlias(String line) {
    String[] ruleArgs = line.split("\\s+");
    if (aliases == null) {
      // first line should be the aliases count
      final int count = Integer.parseInt(ruleArgs[1]);
      aliases = new String[count];
    } else {
      // an alias can map to no flags
      String aliasValue = ruleArgs.length == 1 ? "" : ruleArgs[1];
      aliases[aliasCount++] = aliasValue;
    }
  }

  private String getAliasValue(int id) {
    try {
      return aliases[id - 1];
    } catch (IndexOutOfBoundsException ex) {
      throw new IllegalArgumentException("Bad flag alias number:" + id, ex);
    }
  }

  private void parseMorphAlias(String line) {
    if (morphAliases == null) {
      // first line should be the aliases count
      final int count = Integer.parseInt(line.substring(3));
      morphAliases = new String[count];
    } else {
      String arg = line.substring(2); // leave the space
      morphAliases[morphAliasCount++] = arg;
    }
  }

  private List<String> splitMorphData(String morphData) {
    // first see if it's an alias
    if (morphAliasCount > 0) {
      try {
        int alias = Integer.parseInt(morphData.trim());
        morphData = morphAliases[alias - 1];
      } catch (NumberFormatException ignored) {
      }
    }
    if (morphData.isBlank()) {
      return Collections.emptyList();
    }
    return Arrays.stream(morphData.split("\\s+"))
        .filter(
            s ->
                s.length() > 3
                    && Character.isLetter(s.charAt(0))
                    && Character.isLetter(s.charAt(1))
                    && s.charAt(2) == ':')
        .collect(Collectors.toList());
  }

  boolean hasFlag(IntsRef forms, char flag) {
    int formStep = formStep();
    for (int i = 0; i < forms.length; i += formStep) {
      if (hasFlag(forms.ints[forms.offset + i], flag)) {
        return true;
      }
    }
    return false;
  }

  /** Abstraction of the process of parsing flags taken from the affix and dic files */
  abstract static class FlagParsingStrategy {
    // we don't check the flag count, as Hunspell accepts longer sequences
    // https://github.com/hunspell/hunspell/issues/707
    static final boolean checkFlags = false;

    /**
     * Parses the given String into a single flag
     *
     * @param rawFlag String to parse into a flag
     * @return Parsed flag
     */
    char parseFlag(String rawFlag) {
      char[] flags = parseFlags(rawFlag);
      if (checkFlags && flags.length != 1) {
        throw new IllegalArgumentException("expected only one flag, got: " + rawFlag);
      }
      return flags[0];
    }

    /**
     * Parses the given String into multiple flags
     *
     * @param rawFlags String to parse into flags
     * @return Parsed flags
     */
    abstract char[] parseFlags(String rawFlags);
  }

  /**
   * Simple implementation of {@link FlagParsingStrategy} that treats the chars in each String as a
   * individual flags. Can be used with both the ASCII and UTF-8 flag types.
   */
  private static class SimpleFlagParsingStrategy extends FlagParsingStrategy {
    @Override
    public char[] parseFlags(String rawFlags) {
      return rawFlags.toCharArray();
    }
  }

  /** Used to read flags as UTF-8 even if the rest of the file is in the default (8-bit) encoding */
  private static class DefaultAsUtf8FlagParsingStrategy extends FlagParsingStrategy {
    @Override
    public char[] parseFlags(String rawFlags) {
      return new String(rawFlags.getBytes(DEFAULT_CHARSET), StandardCharsets.UTF_8).toCharArray();
    }
  }

  /**
   * Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded in its
   * numerical form. In the case of multiple flags, each number is separated by a comma.
   */
  private static class NumFlagParsingStrategy extends FlagParsingStrategy {
    @Override
    public char[] parseFlags(String rawFlags) {
      StringBuilder result = new StringBuilder();
      StringBuilder group = new StringBuilder();
      for (int i = 0; i <= rawFlags.length(); i++) {
        if (i == rawFlags.length() || rawFlags.charAt(i) == ',') {
          if (group.length() > 0) { // ignoring empty flags (this happens in danish, for example)
            int flag = Integer.parseInt(group, 0, group.length(), 10);
            if (flag >= DEFAULT_FLAGS) {
              // accept 0 due to https://github.com/hunspell/hunspell/issues/708
              throw new IllegalArgumentException(
                  "Num flags should be between 0 and " + DEFAULT_FLAGS + ", found " + flag);
            }
            result.append((char) flag);
            group.setLength(0);
          }
        } else if (rawFlags.charAt(i) >= '0' && rawFlags.charAt(i) <= '9') {
          group.append(rawFlags.charAt(i));
        }
      }

      return result.toString().toCharArray();
    }
  }

  /**
   * Implementation of {@link FlagParsingStrategy} that assumes each flag is encoded as two ASCII
   * characters whose codes must be combined into a single character.
   */
  private static class DoubleASCIIFlagParsingStrategy extends FlagParsingStrategy {

    @Override
    public char[] parseFlags(String rawFlags) {
      if (checkFlags && rawFlags.length() % 2 == 1) {
        throw new IllegalArgumentException(
            "Invalid flags (should be even number of characters): " + rawFlags);
      }

      char[] flags = new char[rawFlags.length() / 2];
      for (int i = 0; i < flags.length; i++) {
        char f1 = rawFlags.charAt(i * 2);
        char f2 = rawFlags.charAt(i * 2 + 1);
        if (f1 >= 256 || f2 >= 256) {
          throw new IllegalArgumentException(
              "Invalid flags (LONG flags must be double ASCII): " + rawFlags);
        }
        flags[i] = (char) (f1 << 8 | f2);
      }
      return flags;
    }
  }

  boolean hasFlag(int entryId, char flag) {
    return flagLookup.hasFlag(entryId, flag);
  }

  boolean mayNeedInputCleaning() {
    return ignoreCase || ignore != null || iconv != null;
  }

  boolean needsInputCleaning(CharSequence input) {
    if (mayNeedInputCleaning()) {
      for (int i = 0; i < input.length(); i++) {
        char ch = input.charAt(i);
        if (ignore != null && Arrays.binarySearch(ignore, ch) >= 0
            || ignoreCase && caseFold(ch) != ch
            || iconv != null && iconv.mightReplaceChar(ch)) {
          return true;
        }
      }
    }
    return false;
  }

  CharSequence cleanInput(CharSequence input, StringBuilder reuse) {
    reuse.setLength(0);

    for (int i = 0; i < input.length(); i++) {
      char ch = input.charAt(i);

      if (ignore != null && Arrays.binarySearch(ignore, ch) >= 0) {
        continue;
      }

      if (ignoreCase && iconv == null) {
        // if we have no input conversion mappings, do this on-the-fly
        ch = caseFold(ch);
      }

      reuse.append(ch);
    }

    if (iconv != null) {
      iconv.applyMappings(reuse);
      if (ignoreCase) {
        for (int i = 0; i < reuse.length(); i++) {
          reuse.setCharAt(i, caseFold(reuse.charAt(i)));
        }
      }
    }

    return reuse;
  }

  private static char[] toSortedCharArray(Set<Character> set) {
    char[] chars = new char[set.size()];
    int i = 0;
    for (Character c : set) {
      chars[i++] = c;
    }
    Arrays.sort(chars);
    return chars;
  }

  boolean isSecondStagePrefix(char flag) {
    return Arrays.binarySearch(secondStagePrefixFlags, flag) >= 0;
  }

  boolean isSecondStageSuffix(char flag) {
    return Arrays.binarySearch(secondStageSuffixFlags, flag) >= 0;
  }

  /** folds single character (according to LANG if present) */
  char caseFold(char c) {
    if (alternateCasing) {
      if (c == 'I') {
        return 'ı';
      } else if (c == 'İ') {
        return 'i';
      } else {
        return Character.toLowerCase(c);
      }
    } else {
      return Character.toLowerCase(c);
    }
  }

  /** Returns true if this dictionary was constructed with the {@code ignoreCase} option */
  public boolean getIgnoreCase() {
    return ignoreCase;
  }

  /**
   * Returns the default temporary directory pointed to by {@code java.io.tmpdir}. If not accessible
   * or not available, an IOException is thrown.
   */
  static Path getDefaultTempDir() throws IOException {
    String tmpDir = System.getProperty("java.io.tmpdir");
    if (tmpDir == null) {
      throw new IOException("No temporary path (java.io.tmpdir)?");
    }
    Path tmpPath = Paths.get(tmpDir);
    if (!Files.isWritable(tmpPath)) {
      throw new IOException(
          "Temporary path not present or writeable?: " + tmpPath.toAbsolutePath());
    }
    return tmpPath;
  }

  /** Possible word breaks according to BREAK directives */
  static class Breaks {
    private static final Set<String> MINUS = Collections.singleton("-");
    static final Breaks DEFAULT = new Breaks(MINUS, MINUS, MINUS);
    final String[] starting, ending, middle;

    Breaks(Collection<String> starting, Collection<String> ending, Collection<String> middle) {
      this.starting = starting.toArray(new String[0]);
      this.ending = ending.toArray(new String[0]);
      this.middle = middle.toArray(new String[0]);
    }

    boolean isNotEmpty() {
      return middle.length > 0 || starting.length > 0 || ending.length > 0;
    }
  }
}