/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nutch.indexer.replace;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.indexer.basic.BasicIndexingFilter;
import org.apache.nutch.indexer.metadata.MetadataIndexer;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseUtil;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolFactory;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Assert;
import org.junit.Test;

/**
 * JUnit tests for the <code>index-replace</code> plugin.
 * 
 * In these tests, the sample file has some meta tags added to the Nutch
 * document by the <code>index-metadata</code> plugin. The
 * <code>index-replace</code> plugin is then used to either change (or not
 * change) the fields depending on the various values of
 * <code>index.replace.regexp</code> property being provided to Nutch.
 * 
 * 
 * @author Peter Ciuffetti
 *
 */
public class TestIndexReplace {

  private static final String INDEX_REPLACE_PROPERTY = "index.replace.regexp";

  private String fileSeparator = System.getProperty("file.separator");
  private String sampleDir = System.getProperty("test.data", ".");
  private String sampleFile = "testIndexReplace.html";

  /**
   * Run a test file through the Nutch parser and index filters.
   * 
   * @param fileName
   * @param conf
   * @return the Nutch document with the replace indexer applied
   */
  public NutchDocument parseAndFilterFile(String fileName, Configuration conf) {
    NutchDocument doc = new NutchDocument();

    BasicIndexingFilter basicIndexer = new BasicIndexingFilter();
    basicIndexer.setConf(conf);
    Assert.assertNotNull(basicIndexer);

    MetadataIndexer metaIndexer = new MetadataIndexer();
    metaIndexer.setConf(conf);
    Assert.assertNotNull(basicIndexer);

    ReplaceIndexer replaceIndexer = new ReplaceIndexer();
    replaceIndexer.setConf(conf);
    Assert.assertNotNull(replaceIndexer);

    try {
      String urlString = "file:" + sampleDir + fileSeparator + fileName;
      Text text = new Text(urlString);
      CrawlDatum crawlDatum = new CrawlDatum();
      Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString);
      Content content = protocol.getProtocolOutput(text, crawlDatum)
          .getContent();
      Parse parse = new ParseUtil(conf).parse(content).get(content.getUrl());
      crawlDatum.setFetchTime(100L);

      Inlinks inlinks = new Inlinks();
      doc = basicIndexer.filter(doc, parse, text, crawlDatum, inlinks);
      doc = metaIndexer.filter(doc, parse, text, crawlDatum, inlinks);
      doc = replaceIndexer.filter(doc, parse, text, crawlDatum, inlinks);
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail(e.toString());
    }

    return doc;
  }

  /**
   * Test property parsing.
   * 
   * The filter does not expose details of the parse. So all we are checking is
   * that the parse does not throw a runtime exception and that the value
   * provided is the value returned.
   */
  @Test
  public void testPropertyParse() {
    Configuration conf = NutchConfiguration.create();
    String indexReplaceProperty = "  metatag.description=/this(.*)plugin/this awesome plugin/2\n"
        + "  metatag.keywords=/\\,/\\!/\n"
        + "  hostmatch=.*.com\n"
        + "  metatag.keywords=/\\,/\\?/\n"
        + "  metatag.author:dc_author=/\\s+/ David /\n"
        + "  urlmatch=.*.html\n"
        + "  metatag.keywords=/\\,/\\./\n" + "  metatag.author=/\\s+/ D. /\n";

    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);

    ReplaceIndexer rp = new ReplaceIndexer();
    try {
      rp.setConf(conf);
    } catch (RuntimeException ohno) {
      Assert.fail("Unable to parse a valid index.replace.regexp property! "
          + ohno.getMessage());
    }

    Configuration parsedConf = rp.getConf();

    // Does the getter equal the setter? Too easy!
    Assert.assertEquals(indexReplaceProperty,
        parsedConf.get(INDEX_REPLACE_PROPERTY));
  }

  /**
   * Test metatag value replacement using global replacement settings.
   * 
   * The index.replace.regexp property does not use hostmatch or urlmatch, so
   * all patterns are global.
   */
  @Test
  public void testGlobalReplacement() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking! Riveting! Two Thumbs Up!";
    String expectedAuthor = "Peter D. Ciuffetti";
    String indexReplaceProperty = "  metatag.description=/this(.*)plugin/this awesome plugin/\n"
        + "  metatag.keywords=/\\,/\\!/\n" + "  metatag.author=/\\s+/ D. /\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));
  }

  /**
   * Test that invalid property settings are handled and ignored.
   * 
   * This test provides an invalid property setting that will fail property
   * parsing and Pattern.compile. The expected outcome is that the patterns will
   * not cause failure and the targeted fields will not be modified by the
   * filter.
   */
  @Test
  public void testInvalidPatterns() {
    String expectedDescription = "With this plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking, Riveting, Two Thumbs Up!";
    String expectedAuthor = "Peter Ciuffetti";
    // Contains: invalid pattern, invalid flags, incomplete property
    String indexReplaceProperty = "  metatag.description=/this\\s+**plugin/this awesome plugin/\n"
        + "  metatag.keywords=/\\,/\\!/what\n" + " metatag.author=#notcomplete";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Assert that our metatags have not changed.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));

  }

  /**
   * Test URL pattern matching
   */
  @Test
  public void testUrlMatchesPattern() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking! Riveting! Two Thumbs Up!";
    String expectedAuthor = "Peter D. Ciuffetti";
    String indexReplaceProperty = " urlmatch=.*.html\n"
        + "  metatag.description=/this(.*)plugin/this awesome plugin/\n"
        + "  metatag.keywords=/\\,/\\!/\n" + "  metatag.author=/\\s+/ D. /\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Assert that our metatags have changed.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));

  }

  /**
   * Test URL pattern not matching.
   * 
   * Expected result is that the filter does not change the fields.
   */
  @Test
  public void testUrlNotMatchesPattern() {
    String expectedDescription = "With this plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking, Riveting, Two Thumbs Up!";
    String expectedAuthor = "Peter Ciuffetti";
    String indexReplaceProperty = " urlmatch=.*.xml\n"
        + "  metatag.description=/this(.*)plugin/this awesome plugin/\n"
        + "  metatag.keywords=/\\,/\\!/\n" + "  metatag.author=/\\s+/ D. /\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Assert that our metatags have not changed.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));

  }

  /**
   * Test a global pattern match for description and URL pattern match for
   * keywords and author.
   * 
   * All three should be triggered. It also tests replacement groups.
   */
  @Test
  public void testGlobalAndUrlMatchesPattern() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking! Riveting! Two Thumbs Up!";
    String expectedAuthor = "Peter D. Ciuffetti";
    String indexReplaceProperty = "  metatag.description=/this(.*)plugin/this$1awesome$1plugin/\n"
        + "  urlmatch=.*.html\n"
        + "  metatag.keywords=/\\,/\\!/\n"
        + "  metatag.author=/\\s+/ D. /\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Assert that our metatags have changed.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));

  }

  /**
   * Test a global pattern match for description and URL pattern match for
   * keywords and author.
   * 
   * Only the global match should be triggered.
   */
  @Test
  public void testGlobalAndUrlNotMatchesPattern() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String expectedKeywords = "Breathtaking, Riveting, Two Thumbs Up!";
    String expectedAuthor = "Peter Ciuffetti";
    String indexReplaceProperty = "  metatag.description=/this(.*)plugin/this$1awesome$1plugin/\n"
        + "  urlmatch=.*.xml\n"
        + "  metatag.keywords=/\\,/\\!/\n"
        + "  metatag.author=/\\s+/ D. /\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Assert that description has changed and the others have not changed.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    Assert
        .assertEquals(expectedKeywords, doc.getFieldValue("metatag.keywords"));
    Assert.assertEquals(expectedAuthor, doc.getFieldValue("metatag.author"));
  }

  /**
   * Test order-specific replacement settings.
   * 
   * This makes multiple replacements on the same field and will produce the
   * expected value only if the replacements are run in the order specified.
   */
  @Test
  public void testReplacementsRunInSpecifedOrder() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String indexReplaceProperty = "  metatag.description=/this plugin/this amazing plugin/\n"
        + "  metatag.description=/this amazing plugin/this valuable plugin/\n"
        + "  metatag.description=/this valuable plugin/this cool plugin/\n"
        + "  metatag.description=/this cool plugin/this wicked plugin/\n"
        + "  metatag.description=/this wicked plugin/this awesome plugin/\n";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Check that the value produced by the last replacement has worked.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
  }

  /**
   * Test a replacement pattern that uses the flags feature.
   * 
   * A 2 is Pattern.CASE_INSENSITIVE. We look for upper case and expect to match
   * any case.
   */
  @Test
  public void testReplacementsWithFlags() {
    String expectedDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String indexReplaceProperty = "  metatag.description=/THIS PLUGIN/this awesome plugin/2";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Check that the value produced by the case-insensitive replacement has
    // worked.
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
  }

  /**
   * Test a replacement pattern that uses the target field feature.
   * Check that the input is not modifid and that the taret field is added.
   */
  @Test
  public void testReplacementsDifferentTarget() {
    String expectedDescription = "With this plugin, I control the description! Bwuhuhuhaha!";
    String expectedTargetDescription = "With this awesome plugin, I control the description! Bwuhuhuhaha!";
    String indexReplaceProperty = "  metatag.description:new=/this plugin/this awesome plugin/";

    Configuration conf = NutchConfiguration.create();
    conf.set(
        "plugin.includes",
        "protocol-file|urlfilter-regex|parse-(html|metatags)|index-(basic|anchor|metadata|static|replace)|urlnormalizer-(pass|regex|basic)");
    conf.set(INDEX_REPLACE_PROPERTY, indexReplaceProperty);
    conf.set("metatags.names", "author,description,keywords");
    conf.set("index.parse.md",
        "metatag.author,metatag.description,metatag.keywords");
    // Not necessary but helpful when debugging the filter.
    conf.set("http.timeout", "99999999999");

    // Run the document through the parser and index filters.
    NutchDocument doc = parseAndFilterFile(sampleFile, conf);

    // Check that the input field has not been modified
    Assert.assertEquals(expectedDescription,
        doc.getFieldValue("metatag.description"));
    // Check that the output field has created
    Assert.assertEquals(expectedTargetDescription,
        doc.getFieldValue("new"));
  }
}