| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.cocoon.components.search; |
| |
| import org.apache.avalon.framework.activity.Disposable; |
| import org.apache.avalon.framework.configuration.Configurable; |
| import org.apache.avalon.framework.configuration.Configuration; |
| import org.apache.avalon.framework.configuration.ConfigurationException; |
| import org.apache.avalon.framework.logger.AbstractLogEnabled; |
| import org.apache.avalon.framework.service.ServiceException; |
| import org.apache.avalon.framework.service.ServiceManager; |
| import org.apache.avalon.framework.service.Serviceable; |
| import org.apache.cocoon.ProcessingException; |
| import org.apache.cocoon.components.crawler.CocoonCrawler; |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.index.IndexReader; |
| import org.apache.lucene.index.IndexWriter; |
| import org.apache.lucene.index.Term; |
| import org.apache.lucene.index.TermEnum; |
| import org.apache.lucene.store.Directory; |
| |
| import java.io.IOException; |
| import java.net.URL; |
| import java.util.Iterator; |
| |
| /** |
| * A lucene indexer. |
| * |
| * <p> |
| * XML documents are indexed using lucene. |
| * Links to XML documents are supplied by |
| * a crawler, requesting links of documents by specifying a cocoon-view, and |
| * HTTP protocol. |
| * </p> |
| * |
| * @author <a href="mailto:berni_huber@a1.net">Bernhard Huber</a> |
| * @version CVS $Id$ |
| */ |
| public class SimpleLuceneCocoonIndexerImpl extends AbstractLogEnabled |
| implements LuceneCocoonIndexer, Configurable, Serviceable, Disposable |
| { |
| |
| /** |
| * configuration tagname for specifying the analyzer class |
| */ |
| public final static String ANALYZER_CLASSNAME_CONFIG = "analyzer-classname"; |
| |
| /** |
| * configuration default analyzer class |
| */ |
| public final static String ANALYZER_CLASSNAME_DEFAULT = "org.apache.lucene.analysis.standard.StandardAnalyzer"; |
| |
| /** |
| * configuration tagname for specifying lucene's index directory |
| */ |
| public final static String DIRECTORY_CONFIG = "directory"; |
| |
| /** |
| * configuration default directory, ie. no default. |
| */ |
| public final static String DIRECTORY_DEFAULT = null; |
| |
| /** |
| * configuration tagname for specifying lucene's merge factor. |
| */ |
| public final static String MERGE_FACTOR_CONFIG = "merge-factor"; |
| |
| /** |
| * configuration default value for |
| * <a href="http://www.mail-archive.com/lucene-user@jakarta.apache.org/msg00373.html">lucene's merge factor</a>. |
| */ |
| public final static int MERGE_FACTOR_DEFAULT = 10; |
| |
| /** |
| * The service manager for looking up components used. |
| */ |
| protected ServiceManager manager = null; |
| |
| /** The used lucene analyzer */ |
| protected Analyzer analyzer; |
| |
| // private String analyzerClassnameDefault = ANALYZER_CLASSNAME_DEFAULT; |
| |
| /** The Lucene Merge Factor */ |
| private int mergeFactor = MERGE_FACTOR_DEFAULT; |
| |
| |
| /** |
| * Sets the analyzer attribute of the SimpleLuceneCocoonIndexerImpl object |
| * |
| * @param analyzer |
| * The new analyzer value |
| */ |
| public void setAnalyzer(Analyzer analyzer) { |
| this.analyzer = analyzer; |
| } |
| |
| |
| /** |
| * Configure this component. |
| * |
| * @param conf is the configuration |
| * @exception ConfigurationException is thrown if configuring fails |
| */ |
| public void configure(Configuration conf) throws ConfigurationException { |
| Configuration child; |
| |
| /* child = conf.getChild(ANALYZER_CLASSNAME_CONFIG, false); |
| if (child != null) { |
| // fix Bugzilla Bug 25277, use child.getValue |
| // and in all following blocks |
| String value = child.getValue(ANALYZER_CLASSNAME_DEFAULT); |
| if (value != null) { |
| analyzerClassnameDefault = value; |
| } |
| } |
| */ |
| child = conf.getChild(MERGE_FACTOR_CONFIG, false); |
| if (child != null) { |
| // fix Bugzilla Bug 25277, use child instead of conf |
| int int_value = child.getValueAsInteger(MERGE_FACTOR_DEFAULT); |
| mergeFactor = int_value; |
| } |
| } |
| |
| |
| /** |
| * Set the current <code>ServiceManager</code> instance used by this |
| * <code>Serviceable</code>. |
| * |
| * @param manager used by this component |
| * @exception ServiceException is never thrown |
| */ |
| public void service(ServiceManager manager) throws ServiceException { |
| this.manager = manager; |
| } |
| |
| |
| /** |
| * Dispose this component. |
| */ |
| public void dispose() { } |
| |
| |
| /** |
| * index content of base_url, index content of links from base_url. |
| * |
| * @param index |
| * the lucene store to write the index to |
| * @param create |
| * if true create, or overwrite existing index, else update |
| * existing index. |
| * @param base_url |
| * index content of base_url, and crawl through all its links |
| * recursivly. |
| * @exception ProcessingException |
| * is thrown if indexing fails |
| */ |
| public void index(Directory index, boolean create, URL base_url) throws ProcessingException { |
| |
| IndexWriter writer = null; |
| LuceneXMLIndexer lxi = null; |
| CocoonCrawler cocoonCrawler = null; |
| |
| try { |
| lxi = (LuceneXMLIndexer) manager.lookup(LuceneXMLIndexer.ROLE); |
| |
| writer = new IndexWriter(index, analyzer, create); |
| writer.mergeFactor = this.mergeFactor; |
| |
| cocoonCrawler = (CocoonCrawler) manager.lookup(CocoonCrawler.ROLE); |
| cocoonCrawler.crawl(base_url); |
| |
| Iterator cocoonCrawlerIterator = cocoonCrawler.iterator(); |
| while (cocoonCrawlerIterator.hasNext()) { |
| URL crawl_url = (URL) cocoonCrawlerIterator.next(); |
| // result of fix Bugzilla Bug 25270, in SimpleCocoonCrawlerImpl |
| // check if crawl_url is null |
| if (crawl_url == null) { |
| continue; |
| } else if (!crawl_url.getHost().equals(base_url.getHost()) || |
| crawl_url.getPort() != base_url.getPort()) { |
| |
| // skip urls using different host, or port than host, |
| // or port of base url |
| if (getLogger().isDebugEnabled()) { |
| getLogger().debug("Skipping crawling URL " + crawl_url.toString() + |
| " as base_url is " + base_url.toString()); |
| } |
| continue; |
| } |
| |
| // build lucene documents from the content of the crawl_url |
| Iterator i = lxi.build(crawl_url).iterator(); |
| |
| // add all built lucene documents |
| while (i.hasNext()) { |
| writer.addDocument((Document) i.next()); |
| } |
| } |
| // optimize it |
| writer.optimize(); |
| } catch (IOException ioe) { |
| throw new ProcessingException("IOException in index()", ioe); |
| } catch (ServiceException se) { |
| throw new ProcessingException("Could not lookup service in index()", se); |
| } finally { |
| if (writer != null) { |
| try { |
| writer.close(); |
| } catch (IOException ioe) { |
| } |
| writer = null; |
| } |
| |
| if (lxi != null) { |
| manager.release(lxi); |
| lxi = null; |
| } |
| if (cocoonCrawler != null) { |
| manager.release(cocoonCrawler); |
| cocoonCrawler = null; |
| } |
| } |
| } |
| |
| /** |
| * A document iterator deleting "old" documents form the index. |
| * |
| * TODO: use this class before indexing, in non-creating mode. |
| */ |
| static class DocumentDeletableIterator { |
| private IndexReader reader; |
| // existing index |
| private TermEnum uidIter; |
| |
| // document id iterator |
| |
| /** |
| * Constructor for the DocumentDeletableIterator object |
| * |
| * @param directory |
| * Description of Parameter |
| * @exception IOException |
| * Description of Exception |
| */ |
| public DocumentDeletableIterator(Directory directory) throws IOException { |
| reader = IndexReader.open(directory); |
| // open existing index |
| uidIter = reader.terms(new Term("uid", "")); |
| // init uid iterator |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @exception IOException |
| * Description of Exception |
| */ |
| public void deleteAllStaleDocuments() throws IOException { |
| while (uidIter.term() != null && uidIter.term().field().equals("uid")) { |
| reader.delete(uidIter.term()); |
| uidIter.next(); |
| } |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @param uid |
| * Description of Parameter |
| * @exception IOException |
| * Description of Exception |
| */ |
| public void deleteModifiedDocuments(String uid) throws IOException { |
| while (documentHasBeenModified(uidIter.term(), uid)) { |
| reader.delete(uidIter.term()); |
| uidIter.next(); |
| } |
| if (documentHasNotBeenModified(uidIter.term(), uid)) { |
| uidIter.next(); |
| } |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @exception Throwable |
| * Description of Exception |
| */ |
| protected void finalize() throws Throwable { |
| super.finalize(); |
| if (uidIter != null) { |
| uidIter.close(); |
| // close uid iterator |
| uidIter = null; |
| } |
| if (reader != null) { |
| reader.close(); |
| // close existing index |
| reader = null; |
| } |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @param term |
| * Description of Parameter |
| * @return Description of the Returned Value |
| */ |
| boolean documentIsDeletable(Term term) { |
| return term != null && term.field().equals("uid"); |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @param term |
| * Description of Parameter |
| * @param uid |
| * Description of Parameter |
| * @return Description of the Returned Value |
| */ |
| boolean documentHasBeenModified(Term term, String uid) { |
| return documentIsDeletable(term) && term.text().compareTo(uid) < 0; |
| } |
| |
| /** |
| * Description of the Method |
| * |
| * @param term |
| * Description of Parameter |
| * @param uid |
| * Description of Parameter |
| * @return Description of the Returned Value |
| */ |
| boolean documentHasNotBeenModified(Term term, String uid) { |
| return documentIsDeletable(term) && term.text().compareTo(uid) == 0; |
| } |
| } |
| } |
