indexer-core/src/main/java/org/apache/maven/index/updater/DefaultIndexUpdater.java

package org.apache.maven.index.updater;

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0    
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.TimeZone;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.maven.index.ComponentSupport;
import org.apache.maven.index.context.DocumentFilter;
import org.apache.maven.index.context.IndexUtils;
import org.apache.maven.index.context.IndexingContext;
import org.apache.maven.index.context.NexusAnalyzer;
import org.apache.maven.index.context.NexusIndexWriter;
import org.apache.maven.index.fs.Lock;
import org.apache.maven.index.fs.Locker;
import org.apache.maven.index.incremental.IncrementalHandler;
import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.io.RawInputStreamFacade;

/**
 * A default index updater implementation
 *
 * @author Jason van Zyl
 * @author Eugene Kuleshov
 */
@Singleton
@Named
public class DefaultIndexUpdater
    extends ComponentSupport
    implements IndexUpdater
{

    private final IncrementalHandler incrementalHandler;

    private final List<IndexUpdateSideEffect> sideEffects;

    @Inject
    public DefaultIndexUpdater( final IncrementalHandler handler, final List<IndexUpdateSideEffect> mySideeffects )
    {
        incrementalHandler = handler;
        sideEffects = mySideeffects;
    }

    public IndexUpdateResult fetchAndUpdateIndex( final IndexUpdateRequest updateRequest )
        throws IOException
    {
        IndexUpdateResult result = new IndexUpdateResult();

        IndexingContext context = updateRequest.getIndexingContext();

        ResourceFetcher fetcher = null;

        if ( !updateRequest.isOffline() )
        {
            fetcher = updateRequest.getResourceFetcher();

            // If no resource fetcher passed in, use the wagon fetcher by default
            // and put back in request for future use
            if ( fetcher == null )
            {
                throw new IOException( "Update of the index without provided ResourceFetcher is impossible." );
            }

            fetcher.connect( context.getId(), context.getIndexUpdateUrl() );
        }

        File cacheDir = updateRequest.getLocalIndexCacheDir();
        Locker locker = updateRequest.getLocker();
        Lock lock = locker != null && cacheDir != null ? locker.lock( cacheDir ) : null;
        try
        {
            if ( cacheDir != null )
            {
                LocalCacheIndexAdaptor cache = new LocalCacheIndexAdaptor( cacheDir, result );

                if ( !updateRequest.isOffline() )
                {
                    cacheDir.mkdirs();

                    try
                    {
                        fetchAndUpdateIndex( updateRequest, fetcher, cache );
                        cache.commit();
                    }
                    finally
                    {
                        fetcher.disconnect();
                    }
                }

                fetcher = cache.getFetcher();
            }
            else if ( updateRequest.isOffline() )
            {
                throw new IllegalArgumentException( "LocalIndexCacheDir can not be null in offline mode" );
            }

            try
            {
                if ( !updateRequest.isCacheOnly() )
                {
                    LuceneIndexAdaptor target = new LuceneIndexAdaptor( updateRequest );
                    result.setTimestamp( fetchAndUpdateIndex( updateRequest, fetcher, target ) );
                    target.commit();
                }
            }
            finally
            {
                fetcher.disconnect();
            }
        }
        finally
        {
            if ( lock != null )
            {
                lock.release();
            }
        }

        return result;
    }

    private Date loadIndexDirectory( final IndexUpdateRequest updateRequest, final ResourceFetcher fetcher,
        final boolean merge, final String remoteIndexFile )
        throws IOException
    {
        File indexDir = File.createTempFile( remoteIndexFile, ".dir" );
        indexDir.delete();
        indexDir.mkdirs();

        FSDirectory directory = FSDirectory.open( indexDir );

        BufferedInputStream is = null;

        try
        {
            is = new BufferedInputStream( fetcher.retrieve( remoteIndexFile ) );

            Date timestamp = null;

            if ( remoteIndexFile.endsWith( ".gz" ) )
            {
                timestamp = unpackIndexData( is, directory, //
                                             updateRequest.getIndexingContext() );
            }
            else
            {
                // legacy transfer format
                timestamp = unpackIndexArchive( is, directory, //
                                                updateRequest.getIndexingContext() );
            }

            if ( updateRequest.getDocumentFilter() != null )
            {
                filterDirectory( directory, updateRequest.getDocumentFilter() );
            }

            if ( merge )
            {
                updateRequest.getIndexingContext().merge( directory );
            }
            else
            {
                updateRequest.getIndexingContext().replace( directory );
            }
            if ( sideEffects != null && sideEffects.size() > 0 )
            {
                getLogger().info( IndexUpdateSideEffect.class.getName() + " extensions found: " + sideEffects.size() );
                for ( IndexUpdateSideEffect sideeffect : sideEffects )
                {
                    sideeffect.updateIndex( directory, updateRequest.getIndexingContext(), merge );
                }
            }

            return timestamp;
        }
        finally
        {
            IOUtil.close( is );

            if ( directory != null )
            {
                directory.close();
            }

            try
            {
                FileUtils.deleteDirectory( indexDir );
            }
            catch ( IOException ex )
            {
                // ignore
            }
        }
    }

    /**
     * Unpack legacy index archive into a specified Lucene <code>Directory</code>
     *
     * @param is        a <code>ZipInputStream</code> with index data
     * @param directory Lucene <code>Directory</code> to unpack index data to
     * @return {@link Date} of the index update or null if it can't be read
     */
    public static Date unpackIndexArchive( final InputStream is, final Directory directory,
        final IndexingContext context )
        throws IOException
    {
        File indexArchive = File.createTempFile( "nexus-index", "" );

        File indexDir = new File( indexArchive.getAbsoluteFile().getParentFile(), indexArchive.getName() + ".dir" );

        indexDir.mkdirs();

        FSDirectory fdir = FSDirectory.open( indexDir );

        try
        {
            unpackDirectory( fdir, is );
            copyUpdatedDocuments( fdir, directory, context );

            Date timestamp = IndexUtils.getTimestamp( fdir );
            IndexUtils.updateTimestamp( directory, timestamp );
            return timestamp;
        }
        finally
        {
            IndexUtils.close( fdir );
            indexArchive.delete();
            IndexUtils.delete( indexDir );
        }
    }

    private static void unpackDirectory( final Directory directory, final InputStream is )
        throws IOException
    {
        byte[] buf = new byte[4096];

        ZipEntry entry;

        ZipInputStream zis = null;

        try
        {
            zis = new ZipInputStream( is );

            while ( ( entry = zis.getNextEntry() ) != null )
            {
                if ( entry.isDirectory() || entry.getName().indexOf( '/' ) > -1 )
                {
                    continue;
                }

                IndexOutput io = directory.createOutput( entry.getName() );
                try
                {
                    int n = 0;

                    while ( ( n = zis.read( buf ) ) != -1 )
                    {
                        io.writeBytes( buf, n );
                    }
                }
                finally
                {
                    IndexUtils.close( io );
                }
            }
        }
        finally
        {
            IndexUtils.close( zis );
        }
    }

    private static void copyUpdatedDocuments( final Directory sourcedir, final Directory targetdir,
        final IndexingContext context )
        throws CorruptIndexException, LockObtainFailedException, IOException
    {
        IndexWriter w = null;
        IndexReader r = null;
        try
        {
            r = IndexReader.open( sourcedir );
            w = new NexusIndexWriter( targetdir, new NexusAnalyzer(), true );

            for ( int i = 0; i < r.maxDoc(); i++ )
            {
                if ( !r.isDeleted( i ) )
                {
                    w.addDocument( IndexUtils.updateDocument( r.document( i ), context ) );
                }
            }

            w.optimize();
            w.commit();
        }
        finally
        {
            IndexUtils.close( w );
            IndexUtils.close( r );
        }
    }

    private static void filterDirectory( final Directory directory, final DocumentFilter filter )
        throws IOException
    {
        IndexReader r = null;
        try
        {
            // explicitly RW reader needed
            r = IndexReader.open( directory, false );

            int numDocs = r.maxDoc();

            for ( int i = 0; i < numDocs; i++ )
            {
                if ( r.isDeleted( i ) )
                {
                    continue;
                }

                Document d = r.document( i );

                if ( !filter.accept( d ) )
                {
                    r.deleteDocument( i );
                }
            }
        }
        finally
        {
            IndexUtils.close( r );
        }

        IndexWriter w = null;
        try
        {
            // analyzer is unimportant, since we are not adding/searching to/on index, only reading/deleting
            w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );

            w.optimize();

            w.commit();
        }
        finally
        {
            IndexUtils.close( w );
        }
    }

    private Properties loadIndexProperties( final File indexDirectoryFile, final String remoteIndexPropertiesName )
    {
        File indexProperties = new File( indexDirectoryFile, remoteIndexPropertiesName );

        FileInputStream fis = null;

        try
        {
            Properties properties = new Properties();

            fis = new FileInputStream( indexProperties );

            properties.load( fis );

            return properties;
        }
        catch ( IOException e )
        {
            getLogger().debug( "Unable to read remote properties stored locally", e );
        }
        finally
        {
            IOUtil.close( fis );
        }

        return null;
    }

    private void storeIndexProperties( final File dir, final String indexPropertiesName, final Properties properties )
        throws IOException
    {
        File file = new File( dir, indexPropertiesName );

        if ( properties != null )
        {
            OutputStream os = new BufferedOutputStream( new FileOutputStream( file ) );
            try
            {
                properties.store( os, null );
            }
            finally
            {
                IOUtil.close( os );
            }
        }
        else
        {
            file.delete();
        }
    }

    private Properties downloadIndexProperties( final ResourceFetcher fetcher )
        throws IOException
    {
        InputStream fis = fetcher.retrieve( IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );

        try
        {
            Properties properties = new Properties();

            properties.load( fis );

            return properties;
        }
        finally
        {
            IOUtil.close( fis );
        }
    }

    public Date getTimestamp( final Properties properties, final String key )
    {
        String indexTimestamp = properties.getProperty( key );

        if ( indexTimestamp != null )
        {
            try
            {
                SimpleDateFormat df = new SimpleDateFormat( IndexingContext.INDEX_TIME_FORMAT );
                df.setTimeZone( TimeZone.getTimeZone( "GMT" ) );
                return df.parse( indexTimestamp );
            }
            catch ( ParseException ex )
            {
            }
        }
        return null;
    }

    /**
     * Unpack index data using specified Lucene Index writer
     *
     * @param is  an input stream to unpack index data from
     * @param w   a writer to save index data
     * @param ics a collection of index creators for updating unpacked documents.
     */
    public static Date unpackIndexData( final InputStream is, final Directory d, final IndexingContext context )
        throws IOException
    {
        NexusIndexWriter w = new NexusIndexWriter( d, new NexusAnalyzer(), true );
        try
        {
            IndexDataReader dr = new IndexDataReader( is );

            IndexDataReadResult result = dr.readIndex( w, context );

            return result.getTimestamp();
        }
        finally
        {
            IndexUtils.close( w );
        }
    }

    /**
     * Filesystem-based ResourceFetcher implementation
     */
    public static class FileFetcher
        implements ResourceFetcher
    {

        private final File basedir;

        public FileFetcher( File basedir )
        {
            this.basedir = basedir;
        }

        public void connect( String id, String url )
            throws IOException
        {
            // don't need to do anything
        }

        public void disconnect()
            throws IOException
        {
            // don't need to do anything
        }

        public void retrieve( String name, File targetFile )
            throws IOException, FileNotFoundException
        {
            FileUtils.copyFile( getFile( name ), targetFile );

        }

        public InputStream retrieve( String name )
            throws IOException, FileNotFoundException
        {
            return new FileInputStream( getFile( name ) );
        }

        private File getFile( String name )
        {
            return new File( basedir, name );
        }

    }

    private abstract class IndexAdaptor
    {

        protected final File dir;

        protected Properties properties;

        protected IndexAdaptor( File dir )
        {
            this.dir = dir;
        }

        public abstract Properties getProperties();

        public abstract void storeProperties()
            throws IOException;

        public abstract void addIndexChunk( ResourceFetcher source, String filename )
            throws IOException;

        public abstract Date setIndexFile( ResourceFetcher source, String string )
            throws IOException;

        public Properties setProperties( ResourceFetcher source )
            throws IOException
        {
            this.properties = downloadIndexProperties( source );
            return properties;
        }

        public abstract Date getTimestamp();

        public void commit()
            throws IOException
        {
            storeProperties();
        }
    }

    private class LuceneIndexAdaptor
        extends IndexAdaptor
    {

        private final IndexUpdateRequest updateRequest;

        public LuceneIndexAdaptor( IndexUpdateRequest updateRequest )
        {
            super( updateRequest.getIndexingContext().getIndexDirectoryFile() );
            this.updateRequest = updateRequest;
        }

        public Properties getProperties()
        {
            if ( properties == null )
            {
                properties = loadIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
            }
            return properties;
        }

        public void storeProperties()
            throws IOException
        {
            storeIndexProperties( dir, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE, properties );
        }

        public Date getTimestamp()
        {
            return updateRequest.getIndexingContext().getTimestamp();
        }

        public void addIndexChunk( ResourceFetcher source, String filename )
            throws IOException
        {
            loadIndexDirectory( updateRequest, source, true, filename );
        }

        public Date setIndexFile( ResourceFetcher source, String filename )
            throws IOException
        {
            return loadIndexDirectory( updateRequest, source, false, filename );
        }

        public void commit()
            throws IOException
        {
            super.commit();

            updateRequest.getIndexingContext().commit();
        }

    }

    private class LocalCacheIndexAdaptor
        extends IndexAdaptor
    {

        private static final String CHUNKS_FILENAME = "chunks.lst";

        private static final String CHUNKS_FILE_ENCODING = "UTF-8";

        private final IndexUpdateResult result;

        private final ArrayList<String> newChunks = new ArrayList<String>();

        public LocalCacheIndexAdaptor( File dir, IndexUpdateResult result )
        {
            super( dir );
            this.result = result;
        }

        public Properties getProperties()
        {
            if ( properties == null )
            {
                properties = loadIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE );
            }
            return properties;
        }

        public void storeProperties()
            throws IOException
        {
            storeIndexProperties( dir, IndexingContext.INDEX_REMOTE_PROPERTIES_FILE, properties );
        }

        public Date getTimestamp()
        {
            Properties properties = getProperties();
            if ( properties == null )
            {
                return null;
            }

            Date timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_TIMESTAMP );

            if ( timestamp == null )
            {
                timestamp = DefaultIndexUpdater.this.getTimestamp( properties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
            }

            return timestamp;
        }

        public void addIndexChunk( ResourceFetcher source, String filename )
            throws IOException
        {
            File chunk = new File( dir, filename );
            FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), chunk );
            newChunks.add( filename );
        }

        public Date setIndexFile( ResourceFetcher source, String filename )
            throws IOException
        {
            cleanCacheDirectory( dir );

            result.setFullUpdate( true );

            File target = new File( dir, filename );
            FileUtils.copyStreamToFile( new RawInputStreamFacade( source.retrieve( filename ) ), target );

            return null;
        }

        @Override
        public void commit()
            throws IOException
        {
            File chunksFile = new File( dir, CHUNKS_FILENAME );
            BufferedOutputStream os = new BufferedOutputStream( new FileOutputStream( chunksFile, true ) );
            Writer w = new OutputStreamWriter( os, CHUNKS_FILE_ENCODING );
            try
            {
                for ( String filename : newChunks )
                {
                    w.write( filename + "\n" );
                }
                w.flush();
            }
            finally
            {
                IOUtil.close( w );
                IOUtil.close( os );
            }
            super.commit();
        }

        public List<String> getChunks()
            throws IOException
        {
            ArrayList<String> chunks = new ArrayList<String>();

            File chunksFile = new File( dir, CHUNKS_FILENAME );
            BufferedReader r =
                new BufferedReader( new InputStreamReader( new FileInputStream( chunksFile ), CHUNKS_FILE_ENCODING ) );
            try
            {
                String str;
                while ( ( str = r.readLine() ) != null )
                {
                    chunks.add( str );
                }
            }
            finally
            {
                IOUtil.close( r );
            }
            return chunks;
        }

        public ResourceFetcher getFetcher()
        {
            return new LocalIndexCacheFetcher( dir )
            {
                @Override
                public List<String> getChunks()
                    throws IOException
                {
                    return LocalCacheIndexAdaptor.this.getChunks();
                }
            };
        }
    }

    abstract static class LocalIndexCacheFetcher
        extends FileFetcher
    {

        public LocalIndexCacheFetcher( File basedir )
        {
            super( basedir );
        }

        public abstract List<String> getChunks()
            throws IOException;
    }

    private Date fetchAndUpdateIndex( final IndexUpdateRequest updateRequest, ResourceFetcher source,
        IndexAdaptor target )
        throws IOException
    {
        if ( !updateRequest.isForceFullUpdate() )
        {
            Properties localProperties = target.getProperties();
            Date localTimestamp = null;

            if ( localProperties != null )
            {
                localTimestamp = getTimestamp( localProperties, IndexingContext.INDEX_TIMESTAMP );
            }

            // this will download and store properties in the target, so next run
            // target.getProperties() will retrieve it
            Properties remoteProperties = target.setProperties( source );

            Date updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_TIMESTAMP );

            // If new timestamp is missing, dont bother checking incremental, we have an old file
            if ( updateTimestamp != null )
            {
                List<String> filenames =
                    incrementalHandler.loadRemoteIncrementalUpdates( updateRequest, localProperties, remoteProperties );

                // if we have some incremental files, merge them in
                if ( filenames != null )
                {
                    for ( String filename : filenames )
                    {
                        target.addIndexChunk( source, filename );
                    }

                    return updateTimestamp;
                }
            }
            else
            {
                updateTimestamp = getTimestamp( remoteProperties, IndexingContext.INDEX_LEGACY_TIMESTAMP );
            }

            // fallback to timestamp comparison, but try with one coming from local properties, and if not possible (is
            // null)
            // fallback to context timestamp
            if ( localTimestamp != null )
            {
                // if we have localTimestamp
                // if incremental can't be done for whatever reason, simply use old logic of
                // checking the timestamp, if the same, nothing to do
                if ( updateTimestamp != null && localTimestamp != null && !updateTimestamp.after( localTimestamp ) )
                {
                    return null; // index is up to date
                }
            }
        }
        else
        {
            // create index properties during forced full index download
            target.setProperties( source );
        }

        try
        {
            Date timestamp = target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".gz" );
            if ( source instanceof LocalIndexCacheFetcher )
            {
                // local cache has inverse organization compared to remote indexes,
                // i.e. initial index file and delta chunks to apply on top of it
                for ( String filename : ( (LocalIndexCacheFetcher) source ).getChunks() )
                {
                    target.addIndexChunk( source, filename );
                }
            }
            return timestamp;
        }
        catch ( IOException ex )
        {
            // try to look for legacy index transfer format
            try
            {
                return target.setIndexFile( source, IndexingContext.INDEX_FILE_PREFIX + ".zip" );
            }
            catch ( IOException ex2 )
            {
                getLogger().error( "Fallback to *.zip also failed: " + ex2 ); // do not bother with stack trace

                throw ex; // original exception more likely to be interesting
            }
        }
    }

    /**
     * Cleans specified cache directory. If present, Locker.LOCK_FILE will not be deleted.
     */
    protected void cleanCacheDirectory( File dir )
        throws IOException
    {
        File[] members = dir.listFiles();
        if ( members == null )
        {
            return;
        }

        for ( File member : members )
        {
            if ( !Locker.LOCK_FILE.equals( member.getName() ) )
            {
                FileUtils.forceDelete( member );
            }
        }
    }

}