/*
 * Copyright 2013 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package opennlp.addons.geoentitylinker.indexing;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.Version;

/**
 *
 * Creates two lucene indexes, geonames and usgs for use in GeoEntityLinker
 */
public class GazetteerIndexer {

  public static void main(String[] args) {
    try {
      GazetteerIndexer i = new GazetteerIndexer();
      i.index(new File("C:\\temp\\gazetteers\\geonamesdata\\allcountries\\allCountries.txt"),
              new File("C:\\temp\\gazetteers\\geonamesdata\\countryinfo.txt"),
              new File("C:\\temp\\gazetteers\\geonamesdata\\admin1CodesASCII.txt"),
              new File("C:\\temp\\gazetteers\\usgsdata\\NationalFile_20141202.txt.txt"),
              new File("C:\\temp\\gazetteers\\usgsdata\\GOVT_UNITS_20140601.txt"),
              new File("C:\\temp\\gazetteers\\"),
              new File("C:\\temp\\gazetteers\\newCountryContextFile.txt"),
              new File("C:\\temp\\gazetteers\\regions.txt"));
    } catch (Exception ex) {
      ex.printStackTrace();
    }
  }

  public GazetteerIndexer() {

  }

  public static interface Separable {

    String getSeparator();
  }

  public enum GazType implements Separable {

    GEONAMES {
              @Override
              public String toString() {
                return "/opennlp_geoentitylinker_geonames_idx";
              }

              @Override
              public String getSeparator() {
                return "\t";
              }
            },
    USGS {
              @Override
              public String toString() {
                return "/opennlp_geoentitylinker_usgsgaz_idx";
              }

              @Override
              public String getSeparator() {
                return "\\|";
              }
            }
  }

  /**
   *
   * @param geonamesData the actual Geonames gazetteer data downloaded from
   * here: http://download.geonames.org/export/dump/ then click on this link
   * 'allCountries.zip'
   * @param geoNamesCountryInfo the countryinfo lookup table that can be
   * downloaded from here
   * http://download.geonames.org/export/dump/countryinfo.txt
   * @param geonamesAdmin1CodesASCII The lookup data for the province names for
   * each place found here:
   * http://download.geonames.org/export/dump/admin1CodesASCII.txt highlight the
   * table view, and copy results into a text file. Make sure the tab delimitted
   * format is maintained.
   * @param usgsDataFile the actual USGS gazetteer downloaded from here:
   * http://geonames.usgs.gov/domestic/download_data.htm click on the
   * national_file####.zip link to get all the most recent features
   *
   * @param usgsGovUnitsFile go to here:
   * http://geonames.usgs.gov/domestic/download_data.htm in the section titled
   * "Topical Gazetteers -- File Format" click on the drop down list and select
   * "Government Units". The downloaded file is what you need for this param.
   * @param outputIndexDir where you want the final index. Must be a directory,
   * not an actual file.
   * @param outputCountryContextFile The output countrycontext file. THis is a
   * very important file used inside the GeoEntityLinker to assist in toponym
   * resolution.
   * @param regionsFile this file contains a list of regions in the following
   * format: tab delimitted text with index 0 as the name of the region, index 1
   * as the longitude, and index 2 as the latitude
   * @throws Exception
   */
  public void index(File geonamesData, File geoNamesCountryInfo, File geonamesAdmin1CodesASCII,
          File usgsDataFile, File usgsGovUnitsFile, File outputIndexDir, File outputCountryContextFile, File regionsFile) throws Exception {
    if (!outputIndexDir.isDirectory()) {
      throw new IllegalArgumentException("outputIndexDir must be a directory.");
    }
    if (!geonamesData.exists()) {
      throw new FileNotFoundException("geonames data file does not exist");
    }
    if (!geoNamesCountryInfo.exists()) {
      throw new FileNotFoundException("geoNamesCountryCodes data file does not exist");
    }
    if (!geonamesAdmin1CodesASCII.exists()) {
      throw new FileNotFoundException("geonamesAdmin1CodesASCII data file does not exist");
    }

    if (!usgsDataFile.exists()) {
      throw new FileNotFoundException("usgsDataFile data file does not exist");
    }
    if (!usgsGovUnitsFile.exists()) {
      throw new FileNotFoundException("usgsGovUnitsFile data file does not exist");
    }
    if (!outputIndexDir.exists()) {
      throw new FileNotFoundException("outputIndexDir data file does not exist");
    }
    if (!regionsFile.exists()) {
      throw new FileNotFoundException("regionsFile data file does not exist");
    }

    String indexloc = outputIndexDir.getPath() + "/opennlp_geoentitylinker_gazetteer";
    Directory index = new MMapDirectory(new File(indexloc));
    Analyzer a = new StandardAnalyzer(Version.LUCENE_48, new CharArraySet(Version.LUCENE_48, new ArrayList(), true));
    Map<String, Analyzer> analyMap = new HashMap<>();

    analyMap.put("countrycode", new KeywordAnalyzer());
    analyMap.put("admincode", new KeywordAnalyzer());
    analyMap.put("loctype", new KeywordAnalyzer());
    analyMap.put("countycode", new KeywordAnalyzer());
    analyMap.put("gazsource", new KeywordAnalyzer());
    
    PerFieldAnalyzerWrapper aWrapper
            = new PerFieldAnalyzerWrapper(a, analyMap);

    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_48, aWrapper);

    IndexWriter w = new IndexWriter(index, config);
    USGSProcessor.process(usgsGovUnitsFile, usgsDataFile, outputCountryContextFile, w);

    GeonamesProcessor.process(geoNamesCountryInfo, geonamesAdmin1CodesASCII, geonamesData, outputCountryContextFile, w);

    RegionProcessor.process(regionsFile, outputCountryContextFile, w);
    w.commit();
    w.close();
    System.out.println("\nIndexing complete. Be sure to add '" + indexloc + "' and context file '" + outputCountryContextFile.getPath() + "' to entitylinker.properties file");
  }

}