MINDEXER-99: Performance improvements
Squashed from
https://github.com/apache/maven-indexer/pull/12
diff --git a/indexer-cli/src/main/java/org/apache/maven/index/cli/NexusIndexerCli.java b/indexer-cli/src/main/java/org/apache/maven/index/cli/NexusIndexerCli.java
index 9f33f67..e57a2a7 100644
--- a/indexer-cli/src/main/java/org/apache/maven/index/cli/NexusIndexerCli.java
+++ b/indexer-cli/src/main/java/org/apache/maven/index/cli/NexusIndexerCli.java
@@ -418,7 +418,7 @@
final List<IndexCreator> indexers = getIndexers( cli, plexus );
try (BufferedInputStream is = new BufferedInputStream( new FileInputStream( indexArchive ) ); //
- FSDirectory directory = FSDirectory.open( outputFolder ))
+ FSDirectory directory = FSDirectory.open( outputFolder.toPath() ))
{
DefaultIndexUpdater.unpackIndexData( is, directory, (IndexingContext) Proxy.newProxyInstance(
getClass().getClassLoader(), new Class[] { IndexingContext.class }, new PartialImplementation()
diff --git a/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java b/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
index 5abebc5..d8d7120 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/DefaultIteratorResultSet.java
@@ -368,7 +368,6 @@
Analyzer analyzer = context.getAnalyzer();
TokenStream baseTokenStream = analyzer.tokenStream( field.getKey(), new StringReader( text ) );
- baseTokenStream.reset();
CachingTokenFilter tokenStream = new CachingTokenFilter(baseTokenStream);
@@ -389,9 +388,6 @@
List<String> bestFragments = getBestFragments( hr.getQuery(), formatter, tokenStream, text, 3 );
- tokenStream.end();
- tokenStream.close();
-
return bestFragments;
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/DefaultNexusIndexer.java b/indexer-core/src/main/java/org/apache/maven/index/DefaultNexusIndexer.java
index 0d976a1..5453caa 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/DefaultNexusIndexer.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/DefaultNexusIndexer.java
@@ -279,7 +279,7 @@
IndexingContext tmpContext = null;
try
{
- final FSDirectory directory = FSDirectory.open( tmpDir );
+ final FSDirectory directory = FSDirectory.open( tmpDir.toPath() );
if ( update )
{
IndexUtils.copyDirectory( context.getIndexDirectory(), directory );
diff --git a/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java b/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java
index b00cd96..f47bead 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/DefaultQueryCreator.java
@@ -141,7 +141,7 @@
}
else
{
- QueryParser qp = new QueryParser( Version.LUCENE_46, field, new NexusAnalyzer() );
+ QueryParser qp = new QueryParser( field, new NexusAnalyzer() );
// small cheap trick
// if a query is not "expert" (does not contain field:val kind of expression)
@@ -273,7 +273,7 @@
{
String qpQuery = query.toLowerCase().replaceAll( "\\.", " " ).replaceAll( "/", " " );
// tokenization should happen against the field!
- QueryParser qp = new QueryParser( Version.LUCENE_46, indexerField.getKey(), new NexusAnalyzer() );
+ QueryParser qp = new QueryParser( indexerField.getKey(), new NexusAnalyzer() );
qp.setDefaultOperator( Operator.AND );
return qp.parse( qpQuery );
}
@@ -306,7 +306,7 @@
String qpQuery = query;
// tokenization should happen against the field!
- QueryParser qp = new QueryParser( Version.LUCENE_46, indexerField.getKey(), new NexusAnalyzer() );
+ QueryParser qp = new QueryParser( indexerField.getKey(), new NexusAnalyzer() );
qp.setDefaultOperator( Operator.AND );
// small cheap trick
diff --git a/indexer-core/src/main/java/org/apache/maven/index/DefaultScannerListener.java b/indexer-core/src/main/java/org/apache/maven/index/DefaultScannerListener.java
index 52b94ec..6da05f2 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/DefaultScannerListener.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/DefaultScannerListener.java
@@ -261,7 +261,7 @@
{
for ( String uinfo : uinfos )
{
- TopScoreDocCollector collector = TopScoreDocCollector.create( 1, false );
+ TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
indexSearcher.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
@@ -318,4 +318,4 @@
result.setDeletedFiles( deleted );
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java b/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
index 841bf1c..4bef62b 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/DefaultSearchEngine.java
@@ -330,7 +330,7 @@
if ( AbstractSearchRequest.UNDEFINED != topHitCount )
{
// count is set, simply just execute it as-is
- final TopScoreDocCollector hits = TopScoreDocCollector.create( topHitCount, true );
+ final TopScoreDocCollector hits = TopScoreDocCollector.create( topHitCount );
indexSearcher.search( query, hits );
@@ -342,7 +342,7 @@
topHitCount = 1000;
// perform search
- TopScoreDocCollector hits = TopScoreDocCollector.create( topHitCount, true );
+ TopScoreDocCollector hits = TopScoreDocCollector.create( topHitCount );
indexSearcher.search( query, hits );
// check total hits against, does it fit?
@@ -361,7 +361,7 @@
}
// redo all, but this time with correct numbers
- hits = TopScoreDocCollector.create( topHitCount, true );
+ hits = TopScoreDocCollector.create( topHitCount );
indexSearcher.search( query, hits );
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/AbstractIndexingContext.java b/indexer-core/src/main/java/org/apache/maven/index/context/AbstractIndexingContext.java
index 37ec5c1..36887b4 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/AbstractIndexingContext.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/AbstractIndexingContext.java
@@ -28,7 +28,8 @@
{
try
{
- return getIndexDirectory().fileExists( INDEX_UPDATER_PROPERTIES_FILE );
+ getIndexDirectory().fileLength( INDEX_UPDATER_PROPERTIES_FILE );
+ return true;
}
catch ( IOException e )
{
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java b/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
index afae82b..c7ede33 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/DefaultIndexingContext.java
@@ -21,6 +21,11 @@
import java.io.File;
import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -45,6 +50,9 @@
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.FSLockFactory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Bits;
import org.apache.maven.index.ArtifactInfo;
import org.apache.maven.index.artifact.GavCalculator;
@@ -77,6 +85,8 @@
private Directory indexDirectory;
+ private TrackingLockFactory lockFactory;
+
private File indexDirectoryFile;
private String id;
@@ -111,9 +121,11 @@
File repository, //
String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, Directory indexDirectory,
+ TrackingLockFactory lockFactory,
boolean reclaimIndex )
throws ExistingLuceneIndexMismatchException, IOException
{
+
this.id = id;
this.searchable = true;
@@ -134,6 +146,8 @@
this.indexDirectory = indexDirectory;
+ this.lockFactory = lockFactory;
+
// eh?
// Guice does NOT initialize these, and we have to do manually?
// While in Plexus, all is well, but when in guice-shim,
@@ -150,15 +164,24 @@
setIndexDirectoryFile( null );
}
+ private DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
+ TrackingLockFactory lockFactory, String repositoryUrl, String indexUpdateUrl,
+ List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
+ throws IOException, ExistingLuceneIndexMismatchException
+ {
+ this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
+ FSDirectory.open( indexDirectoryFile.toPath(), lockFactory ), lockFactory, reclaimIndex );
+
+ setIndexDirectoryFile( indexDirectoryFile );
+ }
+
public DefaultIndexingContext( String id, String repositoryId, File repository, File indexDirectoryFile,
String repositoryUrl, String indexUpdateUrl,
List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
- this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators,
- FSDirectory.open( indexDirectoryFile ), reclaimIndex );
-
- setIndexDirectoryFile( indexDirectoryFile );
+ this( id, repositoryId, repository, indexDirectoryFile, new TrackingLockFactory( FSLockFactory.getDefault() ),
+ repositoryUrl, indexUpdateUrl, indexCreators, reclaimIndex );
}
@Deprecated
@@ -167,11 +190,11 @@
List<? extends IndexCreator> indexCreators, boolean reclaimIndex )
throws IOException, ExistingLuceneIndexMismatchException
{
- this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, reclaimIndex );
+ this( id, repositoryId, repository, repositoryUrl, indexUpdateUrl, indexCreators, indexDirectory, null, reclaimIndex ); //Lock factory already installed - pass null
if ( indexDirectory instanceof FSDirectory )
{
- setIndexDirectoryFile(( (FSDirectory) indexDirectory ).getDirectory() );
+ setIndexDirectoryFile(( (FSDirectory) indexDirectory ).getDirectory().toFile() );
}
}
@@ -215,7 +238,7 @@
// unlock the dir forcibly
if ( IndexWriter.isLocked( indexDirectory ) )
{
- IndexWriter.unlock( indexDirectory );
+ unlockForcibly( lockFactory, indexDirectory );
}
openAndWarmup();
@@ -252,7 +275,7 @@
// unlock the dir forcibly
if ( IndexWriter.isLocked( indexDirectory ) )
{
- IndexWriter.unlock( indexDirectory );
+ unlockForcibly( lockFactory, indexDirectory );
}
deleteIndexFiles( true );
@@ -281,7 +304,7 @@
// check for descriptor if this is not a "virgin" index
if ( getSize() > 0 )
{
- final TopScoreDocCollector collector = TopScoreDocCollector.create( 1, false );
+ final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
final IndexSearcher indexSearcher = acquireIndexSearcher();
try
{
@@ -369,15 +392,23 @@
if ( full )
{
- if ( indexDirectory.fileExists( INDEX_PACKER_PROPERTIES_FILE ) )
+ try
{
indexDirectory.deleteFile( INDEX_PACKER_PROPERTIES_FILE );
}
+ catch ( IOException ioe )
+ {
+ //Does not exist
+ }
- if ( indexDirectory.fileExists( INDEX_UPDATER_PROPERTIES_FILE ) )
+ try
{
indexDirectory.deleteFile( INDEX_UPDATER_PROPERTIES_FILE );
}
+ catch ( IOException ioe )
+ {
+ //Does not exist
+ }
}
IndexUtils.deleteTimestamp( indexDirectory );
@@ -546,7 +577,6 @@
public synchronized void optimize()
throws CorruptIndexException, IOException
{
- getIndexWriter().forceMerge(1);
commit();
}
@@ -587,6 +617,12 @@
public synchronized void replace( Directory directory )
throws IOException
{
+ replace( directory, null, null );
+ }
+
+ public synchronized void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
+ throws IOException
+ {
final Date ts = IndexUtils.getTimestamp( directory );
closeReaders();
deleteIndexFiles( false );
@@ -594,7 +630,21 @@
openAndWarmup();
// reclaim the index as mine
storeDescriptor();
- rebuildGroups();
+ if ( allGroups == null && rootGroups == null )
+ {
+ rebuildGroups();
+ }
+ else
+ {
+ if ( allGroups != null )
+ {
+ setAllGroups( allGroups );
+ }
+ if ( rootGroups != null )
+ {
+ setRootGroups( rootGroups );
+ }
+ }
updateTimestamp( true, ts );
optimize();
}
@@ -612,7 +662,7 @@
try
{
final IndexWriter w = getIndexWriter();
- final IndexReader directoryReader = IndexReader.open( directory);
+ final IndexReader directoryReader = DirectoryReader.open( directory );
TopScoreDocCollector collector = null;
try
{
@@ -635,7 +685,7 @@
String uinfo = d.get( ArtifactInfo.UINFO );
if ( uinfo != null )
{
- collector = TopScoreDocCollector.create( 1, false );
+ collector = TopScoreDocCollector.create( 1 );
s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), collector );
if ( collector.getTotalHits() == 0 )
{
@@ -706,7 +756,7 @@
public List<IndexCreator> getIndexCreators()
{
- return Collections.unmodifiableList( indexCreators );
+ return Collections.<IndexCreator>unmodifiableList( indexCreators );
}
// groups
@@ -784,7 +834,7 @@
protected Set<String> getGroups( String field, String filedValue, String listField )
throws IOException, CorruptIndexException
{
- final TopScoreDocCollector collector = TopScoreDocCollector.create( 1, false );
+ final TopScoreDocCollector collector = TopScoreDocCollector.create( 1 );
final IndexSearcher indexSearcher = acquireIndexSearcher();
try
{
@@ -833,4 +883,43 @@
{
return id + " : " + timestamp;
}
+
+ private static void unlockForcibly( final TrackingLockFactory lockFactory, final Directory dir ) throws IOException
+ {
+ //Warning: Not doable in lucene >= 5.3 consider to remove it as IndexWriter.unlock
+ //was always strongly non recommended by Lucene.
+ //For now try to do the best to simulate the IndexWriter.unlock at least on FSDirectory
+ //using FSLockFactory, the RAMDirectory uses SingleInstanceLockFactory.
+ //custom lock factory?
+ if (lockFactory != null) {
+ final Set<? extends Lock> emittedLocks = lockFactory.getEmittedLocks(IndexWriter.WRITE_LOCK_NAME);
+ for (Lock emittedLock : emittedLocks) {
+ emittedLock.close();
+ }
+ }
+ if (dir instanceof FSDirectory) {
+ final FSDirectory fsdir = (FSDirectory) dir;
+ final Path dirPath = fsdir.getDirectory();
+ if (Files.isDirectory(dirPath)) {
+ Path lockPath = dirPath.resolve(IndexWriter.WRITE_LOCK_NAME);
+ try {
+ lockPath = lockPath.toRealPath();
+ } catch (IOException ioe) {
+ //Not locked
+ return;
+ }
+ try (final FileChannel fc = FileChannel.open(lockPath, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) {
+ final FileLock lck = fc.tryLock();
+ if (lck == null) {
+ //Still active
+ throw new LockObtainFailedException("Lock held by another process: " + lockPath);
+ } else {
+ //Not held fine to release
+ lck.close();
+ }
+ }
+ Files.delete(lockPath);
+ }
+ }
+ }
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/IndexUtils.java b/indexer-core/src/main/java/org/apache/maven/index/context/IndexUtils.java
index d10461c..7e68e1f 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/IndexUtils.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/IndexUtils.java
@@ -20,9 +20,11 @@
*/
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.file.NoSuchFileException;
import java.util.Date;
import org.apache.lucene.document.Document;
@@ -50,9 +52,9 @@
//FIXME: check if this copies too much, Lucene 4 has no filter for lucene files
//Directory.copy( source, target, false );
- for (String file : source.listAll())
+ for ( String file : source.listAll() )
{
- source.copy(target, file, file, IOContext.DEFAULT);
+ target.copyFrom( source, file, file, IOContext.DEFAULT );
}
copyFile( source, target, IndexingContext.INDEX_UPDATER_PROPERTIES_FILE );
@@ -71,41 +73,16 @@
public static boolean copyFile( Directory source, Directory target, String srcName, String targetName )
throws IOException
{
- if ( !source.fileExists( srcName ) )
+ try
+ {
+ source.fileLength( srcName ); // instead of fileExists
+ }
+ catch (FileNotFoundException | NoSuchFileException e)
{
return false;
}
-
- byte[] buf = new byte[BUFFER_SIZE];
-
- IndexInput is = null;
- IndexOutput os = null;
-
- try
- {
- is = source.openInput( srcName, IOContext.DEFAULT);
-
- os = target.createOutput( targetName, IOContext.DEFAULT);
-
- // and copy to dest directory
- long len = is.length();
- long readCount = 0;
- while ( readCount < len )
- {
- int toRead = readCount + BUFFER_SIZE > len ? (int) ( len - readCount ) : BUFFER_SIZE;
- is.readBytes( buf, 0, toRead );
- os.writeBytes( buf, toRead );
- readCount += toRead;
- }
-
- return true;
- }
- finally
- {
- close( os );
-
- close( is );
- }
+ target.copyFrom( source, srcName, targetName, IOContext.DEFAULT );
+ return true;
}
// timestamp
@@ -155,12 +132,18 @@
public static Document updateDocument( Document doc, IndexingContext context, boolean updateLastModified )
{
- ArtifactInfo ai = constructArtifactInfo( doc, context );
- if ( ai == null )
- {
- return doc;
- }
+ return updateDocument(doc, context, updateLastModified, null);
+ }
+ public static Document updateDocument( Document doc, IndexingContext context, boolean updateLastModified, ArtifactInfo ai )
+ {
+ if( ai == null ) {
+ ai = constructArtifactInfo( doc, context );
+ if ( ai == null )
+ {
+ return doc;
+ }
+ }
Document document = new Document();
// unique key
@@ -187,10 +170,14 @@
public static void deleteTimestamp( Directory directory )
throws IOException
{
- if ( directory.fileExists( TIMESTAMP_FILE ) )
+ try
{
directory.deleteFile( TIMESTAMP_FILE );
}
+ catch (FileNotFoundException | NoSuchFileException e)
+ {
+ //Does not exist
+ }
}
public static void updateTimestamp( Directory directory, Date timestamp )
@@ -209,8 +196,6 @@
try
{
io.writeLong( timestamp.getTime() );
-
- io.flush();
}
finally
{
@@ -225,26 +210,13 @@
synchronized ( directory )
{
Date result = null;
- try
+ try (IndexInput ii = directory.openInput( TIMESTAMP_FILE, IOContext.DEFAULT)) {
+ result = new Date( ii.readLong() );
+ } catch (FileNotFoundException | NoSuchFileException e) {
+ //Does not exist
+ } catch ( IOException ex )
{
- if ( directory.fileExists( TIMESTAMP_FILE ) )
- {
- IndexInput ii = null;
-
- try
- {
- ii = directory.openInput( TIMESTAMP_FILE, IOContext.DEFAULT);
-
- result = new Date( ii.readLong() );
- }
- finally
- {
- close( ii );
- }
- }
- }
- catch ( IOException ex )
- {
+ //IO failure
}
return result;
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/IndexingContext.java b/indexer-core/src/main/java/org/apache/maven/index/context/IndexingContext.java
index 49a397d..5aaad69 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/IndexingContext.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/IndexingContext.java
@@ -268,6 +268,9 @@
void replace( Directory directory )
throws IOException;
+ void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
+ throws IOException;
+
Directory getIndexDirectory();
File getIndexDirectoryFile();
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/MergedIndexingContext.java b/indexer-core/src/main/java/org/apache/maven/index/context/MergedIndexingContext.java
index fc008ff..45d93c7 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/MergedIndexingContext.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/MergedIndexingContext.java
@@ -79,7 +79,7 @@
boolean searchable, ContextMemberProvider membersProvider )
throws IOException
{
- this( membersProvider, id, repositoryId, repository, FSDirectory.open( indexDirectoryFile ), searchable );
+ this( membersProvider, id, repositoryId, repository, FSDirectory.open( indexDirectoryFile.toPath() ), searchable );
setIndexDirectoryFile( indexDirectoryFile );
}
@@ -93,7 +93,7 @@
if ( indexDirectory instanceof FSDirectory )
{
- setIndexDirectoryFile( ( (FSDirectory) indexDirectory ).getDirectory() );
+ setIndexDirectoryFile( ( (FSDirectory) indexDirectory ).getDirectory().toFile() );
}
}
@@ -282,6 +282,12 @@
// noop
}
+ public void replace( Directory directory, Set<String> allGroups, Set<String> rootGroups )
+ throws IOException
+ {
+ // noop
+ }
+
public Directory getIndexDirectory()
{
return directory;
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusAnalyzer.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusAnalyzer.java
index 02a1e37..45be8b7 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusAnalyzer.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusAnalyzer.java
@@ -21,6 +21,7 @@
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharTokenizer;
import org.apache.lucene.util.Version;
@@ -35,41 +36,52 @@
* @author cstamas
*/
public final class NexusAnalyzer
- extends Analyzer
+ extends AnalyzerWrapper
{
+ private static final Analyzer CLASS_NAMES_ANALYZER = new Analyzer()
+ {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName)
+ {
+ return new TokenStreamComponents(new DeprecatedClassnamesTokenizer());
+ }
+ };
+ private static final Analyzer LETTER_OR_DIGIT_ANALYZER = new Analyzer()
+ {
+ @Override
+ protected TokenStreamComponents createComponents(String filedName)
+ {
+ return new TokenStreamComponents(new LetterOrDigitTokenizer());
+ }
+ };
public NexusAnalyzer()
{
super(PER_FIELD_REUSE_STRATEGY);
}
-
- protected Tokenizer getTokenizer( String fieldName, Reader reader )
+
+ @Override
+ protected Analyzer getWrappedAnalyzer(String fieldName)
{
if ( JarFileContentsIndexCreator.FLD_CLASSNAMES_KW.getKey().equals( fieldName ) )
{
// To keep "backward" compatibility, we have to use old flawed tokenizer.
- return new DeprecatedClassnamesTokenizer( reader );
+ return CLASS_NAMES_ANALYZER;
}
else
{
- return new LetterOrDigitTokenizer( reader );
+ return LETTER_OR_DIGIT_ANALYZER;
}
}
- @Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader)
- {
- return new TokenStreamComponents(getTokenizer(fieldName, reader));
- }
-
// ==
public static class NoopTokenizer
extends CharTokenizer
{
- public NoopTokenizer( Reader in )
+ public NoopTokenizer()
{
- super( Version.LUCENE_46, in );
+ super();
}
@Override
@@ -83,9 +95,9 @@
public static class DeprecatedClassnamesTokenizer
extends CharTokenizer
{
- public DeprecatedClassnamesTokenizer( Reader in )
+ public DeprecatedClassnamesTokenizer()
{
- super( Version.LUCENE_46, in );
+ super();
}
@Override
@@ -104,9 +116,9 @@
public static class LetterOrDigitTokenizer
extends CharTokenizer
{
- public LetterOrDigitTokenizer( Reader in )
+ public LetterOrDigitTokenizer()
{
- super( Version.LUCENE_46, in );
+ super();
}
@Override
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
index 130ee88..90c975f 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java
@@ -29,7 +29,6 @@
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.Version;
/**
* An extension of <a
@@ -43,21 +42,11 @@
IndexWriterConfig create(Analyzer analyzer);
}
- public static IndexWriterConfigFactory CONFIG_FACTORY = new IndexWriterConfigFactory() {
- public IndexWriterConfig create(final Analyzer analyzer) {
- IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, analyzer );
- config.setRAMBufferSizeMB( 2.0 ); // old default
- config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially
- config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT);
- return config;
- }
- };
-
@Deprecated
public NexusIndexWriter( final Directory directory, final Analyzer analyzer, boolean create )
throws CorruptIndexException, LockObtainFailedException, IOException
{
- this(directory, CONFIG_FACTORY.create(analyzer).setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND));
+ this(directory, new IndexWriterConfig(analyzer));
}
public NexusIndexWriter( final Directory directory, final IndexWriterConfig config )
@@ -70,6 +59,11 @@
public static IndexWriterConfig defaultConfig()
{
- return CONFIG_FACTORY.create(new NexusAnalyzer());
+ final IndexWriterConfig config = new IndexWriterConfig( new NexusAnalyzer() );
+ // default open mode is CreateOrAppend which suits us
+ config.setRAMBufferSizeMB( 2.0 ); // old default
+ config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially
+ config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT);
+ return config;
}
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusLegacyAnalyzer.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusLegacyAnalyzer.java
index c574ef6..305baa1 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusLegacyAnalyzer.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusLegacyAnalyzer.java
@@ -19,15 +19,11 @@
* under the License.
*/
-import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.util.CharTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.util.Version;
import org.apache.maven.index.ArtifactInfo;
/**
@@ -38,29 +34,14 @@
* @author Jason van Zyl
*/
public final class NexusLegacyAnalyzer
- extends Analyzer
+ extends AnalyzerWrapper
{
- private static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer( Version.LUCENE_46 );
-
- @Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader)
- {
- try
+ private static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer();
+ private static final Analyzer LETTER_OR_DIGIT_ANALYZER = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(final String fieldName)
{
- return new TokenStreamComponents((Tokenizer) tokenizer(fieldName, reader));
- }
- catch (IOException ex)
- {
- throw new RuntimeException(ex);
- }
- }
-
-
- protected TokenStream tokenizer( String field, final Reader reader ) throws IOException
- {
- if ( !isTextField( field ) )
- {
- return new CharTokenizer(Version.LUCENE_46, reader )
+ return new TokenStreamComponents(new CharTokenizer()
{
@Override
protected boolean isTokenChar(int c )
@@ -73,11 +54,24 @@
{
return Character.toLowerCase( c );
}
- };
+ });
}
- else
+ };
+
+ public NexusLegacyAnalyzer()
+ {
+ super(PER_FIELD_REUSE_STRATEGY);
+ }
+
+ @Override
+ protected Analyzer getWrappedAnalyzer(String fieldName)
+ {
+ if (!isTextField( fieldName ))
{
- return DEFAULT_ANALYZER.tokenStream(field, reader );
+ return LETTER_OR_DIGIT_ANALYZER;
+ } else
+ {
+ return DEFAULT_ANALYZER;
}
}
@@ -87,6 +81,4 @@
|| ArtifactInfo.NAMES.equals( field );
}
-
-
}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/TrackingLockFactory.java b/indexer-core/src/main/java/org/apache/maven/index/context/TrackingLockFactory.java
new file mode 100644
index 0000000..ead848b
--- /dev/null
+++ b/indexer-core/src/main/java/org/apache/maven/index/context/TrackingLockFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.index.context;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+import static com.google.common.base.Preconditions.checkNotNull;
+import java.util.HashSet;
+
+/**
+ *
+ * @author Tomas Zezula
+ */
+final class TrackingLockFactory extends LockFactory {
+
+ private final LockFactory delegate;
+ private final Set<TrackingLock> emittedLocks;
+
+ TrackingLockFactory(final LockFactory delegate) {
+ this.delegate = checkNotNull(delegate);
+ this.emittedLocks = Collections.newSetFromMap(new ConcurrentHashMap<TrackingLock,Boolean>());
+ }
+
+ Set<? extends Lock> getEmittedLocks(String name) {
+ final Set<Lock> result = new HashSet<>();
+ for (TrackingLock lock : emittedLocks) {
+ if (name == null || name.equals(lock.getName())) {
+ result.add(lock);
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public Lock obtainLock(Directory dir, String lockName) throws IOException {
+ final TrackingLock lck = new TrackingLock(
+ delegate.obtainLock(dir, lockName),
+ lockName);
+ emittedLocks.add(lck);
+ return lck;
+ }
+
+
+ private final class TrackingLock extends Lock {
+ private final Lock delegate;
+ private final String name;
+
+ TrackingLock(
+ final Lock delegate,
+ final String name) {
+ this.delegate = checkNotNull(delegate);
+ this.name = checkNotNull(name);
+ }
+
+ String getName() {
+ return name;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ delegate.close();
+ } finally {
+ emittedLocks.remove(this);
+ }
+ }
+
+ @Override
+ public void ensureValid() throws IOException {
+ delegate.ensureValid();
+ }
+ }
+}
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/DefaultIndexUpdater.java b/indexer-core/src/main/java/org/apache/maven/index/updater/DefaultIndexUpdater.java
index 7f90ee1..06b2834 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/DefaultIndexUpdater.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/DefaultIndexUpdater.java
@@ -41,26 +41,16 @@
import java.util.Date;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import java.util.TimeZone;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Bits;
-import org.apache.maven.index.ArtifactInfo;
-import org.apache.maven.index.context.DefaultIndexingContext;
import org.apache.maven.index.context.DocumentFilter;
import org.apache.maven.index.context.IndexUtils;
import org.apache.maven.index.context.IndexingContext;
@@ -71,7 +61,6 @@
import org.apache.maven.index.incremental.IncrementalHandler;
import org.apache.maven.index.updater.IndexDataReader.IndexDataReadResult;
import org.codehaus.plexus.util.FileUtils;
-import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.io.RawInputStreamFacade;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -205,10 +194,14 @@
{
Date timestamp = null;
+ Set<String> rootGroups = null;
+ Set<String> allGroups = null;
if ( remoteIndexFile.endsWith( ".gz" ) )
{
- timestamp = unpackIndexData( is, directory, //
- updateRequest.getIndexingContext() );
+ IndexDataReadResult result = unpackIndexData( is, directory, updateRequest.getIndexingContext() );
+ timestamp = result.getTimestamp();
+ rootGroups = result.getRootGroups();
+ allGroups = result.getAllGroups();
}
else
{
@@ -227,7 +220,7 @@
}
else
{
- updateRequest.getIndexingContext().replace( directory );
+ updateRequest.getIndexingContext().replace( directory, rootGroups, allGroups );
}
if ( sideEffects != null && sideEffects.size() > 0 )
{
@@ -260,7 +253,7 @@
IndexWriter w = null;
try
{
- r = IndexReader.open( directory );
+ r = DirectoryReader.open( directory );
w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
Bits liveDocs = MultiFields.getLiveDocs(r);
@@ -296,7 +289,6 @@
// analyzer is unimportant, since we are not adding/searching to/on index, only reading/deleting
w = new NexusIndexWriter( directory, new NexusAnalyzer(), false );
- w.forceMerge(4);
w.commit();
}
finally
@@ -381,7 +373,7 @@
* @param w a writer to save index data
* @param ics a collection of index creators for updating unpacked documents.
*/
- public static Date unpackIndexData( final InputStream is, final Directory d, final IndexingContext context )
+ public static IndexDataReadResult unpackIndexData( final InputStream is, final Directory d, final IndexingContext context )
throws IOException
{
NexusIndexWriter w = new NexusIndexWriter( d, new NexusAnalyzer(), true );
@@ -389,9 +381,7 @@
{
IndexDataReader dr = new IndexDataReader( is );
- IndexDataReadResult result = dr.readIndex( w, context );
-
- return result.getTimestamp();
+ return dr.readIndex( w, context );
}
finally
{
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/FSDirectoryFactory.java b/indexer-core/src/main/java/org/apache/maven/index/updater/FSDirectoryFactory.java
index aa739ae..9866bbf 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/FSDirectoryFactory.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/FSDirectoryFactory.java
@@ -40,7 +40,7 @@
public FSDirectory open( File indexDir )
throws IOException
{
- return FSDirectory.open( indexDir );
+ return FSDirectory.open( indexDir.toPath() );
}
};
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
index f55fbf9..792943d 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java
@@ -30,6 +30,8 @@
import java.util.zip.GZIPInputStream;
import com.google.common.base.Strings;
+import java.util.LinkedHashSet;
+import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
@@ -51,19 +53,19 @@
public IndexDataReader( final InputStream is )
throws IOException
{
- BufferedInputStream bis = new BufferedInputStream( is, 1024 * 8 );
// MINDEXER-13
// LightweightHttpWagon may have performed automatic decompression
// Handle it transparently
- bis.mark( 2 );
+ is.mark( 2 );
InputStream data;
- if ( bis.read() == 0x1f && bis.read() == 0x8b ) // GZIPInputStream.GZIP_MAGIC
+ if ( is.read() == 0x1f && is.read() == 0x8b ) // GZIPInputStream.GZIP_MAGIC
{
- bis.reset();
- data = new GZIPInputStream( bis, 1024 * 8 );
+ is.reset();
+ data = new BufferedInputStream(new GZIPInputStream( is, 1024 * 8 ), 1024 * 8 );
}
else
{
+ BufferedInputStream bis = new BufferedInputStream( is, 1024 * 8 );
bis.reset();
data = bis;
}
@@ -88,21 +90,32 @@
int n = 0;
Document doc;
+ Set<String> rootGroups = new LinkedHashSet<>();
+ Set<String> allGroups = new LinkedHashSet<>();
+
while ( ( doc = readDocument() ) != null )
{
- w.addDocument( IndexUtils.updateDocument( doc, context, false ) );
+ ArtifactInfo ai = IndexUtils.constructArtifactInfo( doc, context );
+ if(ai != null) {
+ w.addDocument( IndexUtils.updateDocument( doc, context, false, ai ) );
+ rootGroups.add( ai.getRootGroup() );
+ allGroups.add( ai.getGroupId() );
+
+ } else {
+ w.addDocument( doc );
+ }
n++;
}
w.commit();
- w.forceMerge(1);
- w.commit();
-
IndexDataReadResult result = new IndexDataReadResult();
result.setDocumentCount( n );
result.setTimestamp( date );
+ result.setRootGroups( rootGroups );
+ result.setAllGroups( allGroups );
+
return result;
}
@@ -291,6 +304,10 @@
private int documentCount;
+ private Set<String> rootGroups;
+
+ private Set<String> allGroups;
+
public void setDocumentCount( int documentCount )
{
this.documentCount = documentCount;
@@ -311,6 +328,26 @@
return timestamp;
}
+ public void setRootGroups(Set<String> rootGroups)
+ {
+ this.rootGroups = rootGroups;
+ }
+
+ public Set<String> getRootGroups()
+ {
+ return rootGroups;
+ }
+
+ public void setAllGroups(Set<String> allGroups)
+ {
+ this.allGroups = allGroups;
+ }
+
+ public Set<String> getAllGroups()
+ {
+ return allGroups;
+ }
+
}
/**
diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
index 13ad7ce..5d9e338 100644
--- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
+++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataWriter.java
@@ -34,6 +34,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
@@ -242,7 +243,7 @@
public void writeField( IndexableField field )
throws IOException
{
- int flags = ( field.fieldType().indexed() ? F_INDEXED : 0 ) //
+ int flags = ( field.fieldType().indexOptions() != IndexOptions.NONE ? F_INDEXED : 0 ) //
+ ( field.fieldType().tokenized() ? F_TOKENIZED : 0 ) //
+ ( field.fieldType().stored() ? F_STORED : 0 ); //
// + ( false ? F_COMPRESSED : 0 ); // Compressed not supported anymore
diff --git a/indexer-core/src/test/java/org/apache/maven/index/DefaultIndexNexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/DefaultIndexNexusIndexerTest.java
index 99f6183..195b472 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/DefaultIndexNexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/DefaultIndexNexusIndexerTest.java
@@ -181,7 +181,7 @@
File newIndex = new File( getBasedir(), "target/test-new" );
- Directory newIndexDir = FSDirectory.open( newIndex );
+ Directory newIndexDir = FSDirectory.open( newIndex.toPath() );
IndexingContext newContext =
nexusIndexer.addIndexingContext( "test-new", "test", null, newIndexDir, null, null, DEFAULT_CREATORS );
@@ -223,7 +223,7 @@
newContext.close( false );
- newIndexDir = FSDirectory.open( newIndex );
+ newIndexDir = FSDirectory.open( newIndex.toPath());
newContext =
nexusIndexer.addIndexingContext( "test-new", "test", null, newIndexDir, null, null, DEFAULT_CREATORS );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/FSDirectoryDeleteTest.java b/indexer-core/src/test/java/org/apache/maven/index/FSDirectoryDeleteTest.java
index 0912334..5a541ca 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/FSDirectoryDeleteTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/FSDirectoryDeleteTest.java
@@ -53,13 +53,13 @@
nexusIndexer = lookup( NexusIndexer.class );
- indexDir = FSDirectory.open( indexDirFile );
+ indexDir = FSDirectory.open( indexDirFile.toPath() );
context = nexusIndexer.addIndexingContext( "one", "nexus-13", repo, indexDir, null, null, DEFAULT_CREATORS );
nexusIndexer.scan( context );
- otherIndexDir = FSDirectory.open( otherIndexDirFile );
+ otherIndexDir = FSDirectory.open( otherIndexDirFile.toPath() );
otherContext =
nexusIndexer.addIndexingContext( "other", "nexus-13", repo, otherIndexDir, null, null, DEFAULT_CREATORS );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/FullIndexNexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/FullIndexNexusIndexerTest.java
index dc07273..3d006ff 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/FullIndexNexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/FullIndexNexusIndexerTest.java
@@ -356,7 +356,7 @@
File newIndex = new File( getBasedir(), "target/test-new" );
- Directory newIndexDir = FSDirectory.open( newIndex );
+ Directory newIndexDir = FSDirectory.open( newIndex.toPath() );
IndexingContext newContext =
nexusIndexer.addIndexingContext( "test-new", "test", null, newIndexDir, null, null, DEFAULT_CREATORS );
@@ -397,7 +397,7 @@
newContext.close( false );
- newIndexDir = FSDirectory.open( newIndex );
+ newIndexDir = FSDirectory.open( newIndex.toPath() );
newContext =
nexusIndexer.addIndexingContext( "test-new", "test", null, newIndexDir, null, null, DEFAULT_CREATORS );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java b/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
index 4af552c..98dbd9b 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/NexusIndexerTest.java
@@ -118,13 +118,13 @@
// scored search against field having untokenized indexerField only
q = indexer.constructQuery( MAVEN.PACKAGING, "maven-archetype", SearchType.SCORED );
- assertEquals( "p:maven-archetype p:maven-archetype*^0.8", q.toString() );
+ assertEquals( "p:maven-archetype p:maven-archetype*", q.toString() );
// scored search against field having untokenized indexerField only
q = indexer.constructQuery( MAVEN.ARTIFACT_ID, "commons-logging", SearchType.SCORED );
assertEquals(
- "(a:commons-logging a:commons-logging*^0.8) ((+artifactId:commons +artifactId:logging*) artifactId:\"commons logging\")",
+ "(a:commons-logging a:commons-logging*) ((+artifactId:commons +artifactId:logging*) artifactId:\"commons logging\")",
q.toString() );
// scored search against field having tokenized IndexerField only (should be impossible).
diff --git a/indexer-core/src/test/java/org/apache/maven/index/archetype/NexusArchetypeDataSourceTest.java b/indexer-core/src/test/java/org/apache/maven/index/archetype/NexusArchetypeDataSourceTest.java
index 1818d3c..4164f3a 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/archetype/NexusArchetypeDataSourceTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/archetype/NexusArchetypeDataSourceTest.java
@@ -70,7 +70,7 @@
super.deleteDirectory( indexDirFile );
- indexDir = FSDirectory.open( indexDirFile );
+ indexDir = FSDirectory.open( indexDirFile.toPath() );
}
File repo = new File( getBasedir(), "src/test/repo" );
diff --git a/indexer-core/src/test/java/org/apache/maven/index/context/NexusAnalyzerTest.java b/indexer-core/src/test/java/org/apache/maven/index/context/NexusAnalyzerTest.java
index ba78251..a7c71b0 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/context/NexusAnalyzerTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/context/NexusAnalyzerTest.java
@@ -55,7 +55,7 @@
protected void runAndCompare( IndexerField indexerField, String text, String[] expected )
throws IOException
{
- Tokenizer ts = (Tokenizer) nexusAnalyzer.createComponents(indexerField.getKey(), new StringReader( text ) ).getTokenizer();
+ Tokenizer ts = (Tokenizer) nexusAnalyzer.tokenStream(indexerField.getKey(), new StringReader( text ) );
ts.reset();
ArrayList<String> tokenList = new ArrayList<String>();
diff --git a/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java b/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java
new file mode 100644
index 0000000..5590f6c
--- /dev/null
+++ b/indexer-core/src/test/java/org/apache/maven/index/context/TrackingLockFactoryTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.index.context;
+
+import java.io.IOException;
+import java.util.Set;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.SingleInstanceLockFactory;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ *
+ * @author Tomas Zezula
+ */
+public class TrackingLockFactoryTest {
+
+ public TrackingLockFactoryTest() {
+ }
+
+ @Test
+ public void testLockUnlock() throws IOException {
+ final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
+ final RAMDirectory ram = new RAMDirectory(lf);
+ final Lock foo = ram.obtainLock("foo");
+ final Lock bar = ram.obtainLock("bar");
+ bar.close();
+ foo.close();
+ ram.close();
+ }
+
+ @Test
+ public void testLockLocked() throws IOException {
+ final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
+ final RAMDirectory ram = new RAMDirectory(lf);
+ final Lock foo = ram.obtainLock("foo");
+ boolean thrownLOFE = false;
+ try {
+ ram.obtainLock("foo");
+ } catch (LockObtainFailedException e) {
+ thrownLOFE = true;
+ }
+ assertTrue(thrownLOFE);
+ foo.close();
+ final Lock foo2 = ram.obtainLock("foo");
+ foo2.close();
+ ram.close();
+ }
+
+ @Test
+ public void testEmmittedLocks() throws IOException {
+ final TrackingLockFactory lf = new TrackingLockFactory(new SingleInstanceLockFactory());
+ final RAMDirectory ram = new RAMDirectory(lf);
+ final Lock l1 = ram.obtainLock("l1");
+ final Lock l2 = ram.obtainLock("l2");
+ final Lock l3 = ram.obtainLock("l3");
+ l2.close();
+ Set<? extends Lock> emittedLocks = lf.getEmittedLocks(null);
+ assertEquals(2, emittedLocks.size());
+ assertTrue(emittedLocks.contains(l1));
+ assertTrue(emittedLocks.contains(l3));
+ emittedLocks = lf.getEmittedLocks("l3");
+ assertEquals(1, emittedLocks.size());
+ assertTrue(emittedLocks.contains(l3));
+ }
+}
diff --git a/indexer-core/src/test/java/org/apache/maven/index/packer/NEXUS4149TransferFormatTest.java b/indexer-core/src/test/java/org/apache/maven/index/packer/NEXUS4149TransferFormatTest.java
index 8d6b378..b77d6b0 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/packer/NEXUS4149TransferFormatTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/packer/NEXUS4149TransferFormatTest.java
@@ -19,9 +19,11 @@
* under the License.
*/
+import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
@@ -140,7 +142,7 @@
}
// read it up and verify, but stay "low level", directly consume the GZ file and count
- FileInputStream fis = new FileInputStream( new File( packTargetDir, "nexus-maven-repository-index.gz" ) );
+ InputStream fis = new BufferedInputStream( new FileInputStream( new File( packTargetDir, "nexus-maven-repository-index.gz" ) ) );
IndexDataReader reader = new IndexDataReader( fis );
try
{
diff --git a/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java b/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
index cdf6d71..2b89320 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/updater/DefaultIndexUpdaterTest.java
@@ -150,7 +150,11 @@
Collection<ArtifactInfo> tempContent = tempResponse.getResults();
assertEquals( tempContent.toString(), 3, tempContent.size() );
- RAMDirectory tempDir2 = new RAMDirectory( tempContext.getIndexDirectory(), IOContext.DEFAULT );
+ RAMDirectory tempDir2 = new RAMDirectory();
+ for (String file : tempContext.getIndexDirectory().listAll())
+ {
+ tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
+ }
indexer.removeIndexingContext( tempContext, false );
@@ -193,7 +197,11 @@
indexer.deleteArtifactFromIndex(
createArtifactContext( repositoryId, "commons-lang", "commons-lang", "2.4", null ), tempContext );
- RAMDirectory tempDir2 = new RAMDirectory( tempContext.getIndexDirectory(), IOContext.DEFAULT );
+ RAMDirectory tempDir2 = new RAMDirectory();
+ for (String file : tempContext.getIndexDirectory().listAll())
+ {
+ tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
+ }
indexer.removeIndexingContext( tempContext, false );
@@ -267,7 +275,11 @@
indexer.addArtifactToIndex(
createArtifactContext( repositoryId, "org.slf4j.foo", "jcl104-over-slf4j", "1.4.2", null ), context );
- RAMDirectory tempDir2 = new RAMDirectory( tempContext.getIndexDirectory(), IOContext.DEFAULT );
+ RAMDirectory tempDir2 = new RAMDirectory();
+ for (String file : tempContext.getIndexDirectory().listAll())
+ {
+ tempDir2.copyFrom(tempContext.getIndexDirectory(), file, file, IOContext.DEFAULT);
+ }
indexer.removeIndexingContext( tempContext, false );
@@ -408,7 +420,7 @@
with( IndexingContext.INDEX_FILE_PREFIX + ".gz" ) );
will( returnValue( newInputStream( "/index-updater/server-root/nexus-maven-repository-index.gz" ) ) );
- oneOf( tempContext ).replace( with( any( Directory.class ) ) );
+ oneOf( tempContext ).replace( with( any( Directory.class ) ), with( any( Set.class ) ), with( any( Set.class ) ) );
oneOf( mockFetcher ).disconnect();
}
@@ -566,7 +578,7 @@
will( returnValue( newInputStream( "/index-updater/server-root/nexus-maven-repository-index.gz" ) ) );
// could create index archive there and verify that it is merged correctly
- oneOf( tempContext ).replace( with( any( Directory.class ) ) );
+ oneOf( tempContext ).replace( with( any( Directory.class ) ), with( any( Set.class ) ), with( any( Set.class ) ) );
never( mockFetcher ).retrieve( //
with( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
@@ -810,7 +822,7 @@
never( tempContext ).merge( with( any( Directory.class ) ) );
- oneOf( tempContext ).replace( with( any( Directory.class ) ) );
+ oneOf( tempContext ).replace( with( any( Directory.class ) ), with( any( Set.class ) ), with( any( Set.class ) ) );
oneOf( mockFetcher ).disconnect();
}
@@ -970,4 +982,4 @@
{
assertFalse("Index update should have failed, but says it succeeded", updateResult.isSuccessful());
}
-}
\ No newline at end of file
+}
diff --git a/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java b/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
index 91db7f2..65b6b8c 100644
--- a/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
+++ b/indexer-core/src/test/java/org/apache/maven/index/updater/IndexDataTest.java
@@ -29,6 +29,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
@@ -83,7 +84,7 @@
newDir = new RAMDirectory();
- Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context );
+ Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context ).getTimestamp();
assertEquals( timestamp, newTimestamp );
@@ -120,7 +121,7 @@
newDir = new RAMDirectory();
- Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context );
+ Date newTimestamp = DefaultIndexUpdater.unpackIndexData( is, newDir, context ).getTimestamp();
assertEquals( null, newTimestamp );
@@ -134,7 +135,7 @@
Map<String, ArtifactInfo> r1map = readIndex( r1 );
- IndexReader r2 = IndexReader.open( newDir );
+ IndexReader r2 = DirectoryReader.open( newDir );
Map<String, ArtifactInfo> r2map = readIndex( r2 );
diff --git a/indexer-core/src/test/resources/testQueryCreatorNGSearch/case01.txt b/indexer-core/src/test/resources/testQueryCreatorNGSearch/case01.txt
index 17691d9..ece9199 100644
--- a/indexer-core/src/test/resources/testQueryCreatorNGSearch/case01.txt
+++ b/indexer-core/src/test/resources/testQueryCreatorNGSearch/case01.txt
@@ -1,4 +1,4 @@
-### Searched for field urn:maven#groupId (with 2 registered index fields) using query "commons-logg" (QC create LQL "(g:commons-logg g:commons-logg*^0.8) ((+groupId:commons +groupId:logg*) groupId:"commons logg")")
+### Searched for field urn:maven#groupId (with 2 registered index fields) using query "commons-logg" (QC create LQL "(g:commons-logg g:commons-logg*) ((+groupId:commons +groupId:logg*) groupId:"commons logg")")
test :: commons-logging:commons-logging:1.1:null:jar
test :: commons-logging:commons-logging:1.1:sources:jar
test :: commons-logging:commons-logging:1.0.4:null:jar
diff --git a/indexer-core/src/test/resources/testQueryCreatorNGSearch/case05.txt b/indexer-core/src/test/resources/testQueryCreatorNGSearch/case05.txt
index bd52ac6..511516e 100644
--- a/indexer-core/src/test/resources/testQueryCreatorNGSearch/case05.txt
+++ b/indexer-core/src/test/resources/testQueryCreatorNGSearch/case05.txt
@@ -1,4 +1,4 @@
-### Searched for field urn:maven#version (with 2 registered index fields) using query "1.0" (QC create LQL "(v:1.0 v:1.0*^0.8) ((+version:1 +version:0*) version:"1 0")")
+### Searched for field urn:maven#version (with 2 registered index fields) using query "1.0" (QC create LQL "(v:1.0 v:1.0*) ((+version:1 +version:0*) version:"1 0")")
test :: proptest:proptest-archetype:1.0:null:maven-archetype
test :: org.apache.maven.plugins:maven-core-it-plugin:1.0:null:maven-plugin
test :: org.apache.maven.plugins:maven-core-it-plugin:1.0:sources:jar
diff --git a/indexer-examples/indexer-examples-spring/src/main/java/org/apache/maven/indexer/examples/indexing/RepositoryIndexer.java b/indexer-examples/indexer-examples-spring/src/main/java/org/apache/maven/indexer/examples/indexing/RepositoryIndexer.java
index a205288..88f9a27 100644
--- a/indexer-examples/indexer-examples-spring/src/main/java/org/apache/maven/indexer/examples/indexing/RepositoryIndexer.java
+++ b/indexer-examples/indexer-examples-spring/src/main/java/org/apache/maven/indexer/examples/indexing/RepositoryIndexer.java
@@ -61,11 +61,9 @@
private static final Logger LOGGER = LoggerFactory.getLogger( RepositoryIndexer.class );
- private static final Version luceneVersion = Version.LUCENE_48;
-
private static final String[] luceneFields = new String[]{ "g", "a", "v", "p", "c" };
- private static final WhitespaceAnalyzer luceneAnalyzer = new WhitespaceAnalyzer( luceneVersion );
+ private static final WhitespaceAnalyzer luceneAnalyzer = new WhitespaceAnalyzer( );
private Indexer indexer;
@@ -176,7 +174,7 @@
public Set<ArtifactInfo> search( final String queryText )
throws ParseException, IOException
{
- final Query query = new MultiFieldQueryParser( luceneVersion, luceneFields, luceneAnalyzer ).parse( queryText );
+ final Query query = new MultiFieldQueryParser( luceneFields, luceneAnalyzer ).parse( queryText );
LOGGER.debug( "Executing search query: {}; ctx id: {}; idx dir: {}",
new String[]{ query.toString(), indexingContext.getId(),
diff --git a/pom.xml b/pom.xml
index 0bffd83..bf5e15e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,7 +90,7 @@
<eclipse-sisu.version>0.2.1</eclipse-sisu.version>
<sisu-guice.version>3.2.2</sisu-guice.version>
- <lucene.version>4.8.1</lucene.version>
+ <lucene.version>5.5.4</lucene.version>
<maven.version>3.0.5</maven.version>
<aether.version>1.0.0.v20140518</aether.version>
