| package org.apache.maven.index.cli; |
| |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| import java.io.BufferedInputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.IOException; |
| import java.lang.reflect.Proxy; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.List; |
| |
| import org.apache.commons.cli.CommandLine; |
| import org.apache.commons.cli.HelpFormatter; |
| import org.apache.commons.cli.OptionBuilder; |
| import org.apache.commons.cli.Options; |
| import org.apache.commons.cli.ParseException; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.store.FSDirectory; |
| import org.apache.maven.index.ArtifactContext; |
| import org.apache.maven.index.ArtifactInfo; |
| import org.apache.maven.index.ArtifactScanningListener; |
| import org.apache.maven.index.NexusIndexer; |
| import org.apache.maven.index.ScanningResult; |
| import org.apache.maven.index.context.IndexCreator; |
| import org.apache.maven.index.context.IndexingContext; |
| import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; |
| import org.apache.maven.index.packer.IndexPacker; |
| import org.apache.maven.index.packer.IndexPackingRequest; |
| import org.apache.maven.index.packer.IndexPackingRequest.IndexFormat; |
| import org.apache.maven.index.updater.DefaultIndexUpdater; |
| import org.codehaus.plexus.DefaultContainerConfiguration; |
| import org.codehaus.plexus.DefaultPlexusContainer; |
| import org.codehaus.plexus.PlexusConstants; |
| import org.codehaus.plexus.PlexusContainer; |
| import org.codehaus.plexus.classworlds.ClassWorld; |
| import org.codehaus.plexus.component.repository.exception.ComponentLookupException; |
| import org.codehaus.plexus.logging.Logger; |
| import org.codehaus.plexus.logging.LoggerManager; |
| import org.codehaus.plexus.tools.cli.AbstractCli; |
| import org.codehaus.plexus.util.IOUtil; |
| |
| /** |
| * A command line tool that can be used to index local Maven repository. |
| * <p/> |
| * The following command line options are supported: |
| * <ul> |
| * <li>-repository <path> : required path to repository to be indexed</li> |
| * <li>-index <path> : required index folder used to store created index or where previously created index is stored</li> |
| * <li>-name <path> : required repository name/id</li> |
| * <li>-target <path> : optional folder name where to save produced index files</li> |
| * <li>-type <path> : optional indexer types</li> |
| * <li>-format <path> : optional indexer formats</li> |
| * </ul> |
| * When index folder contains previously created index, the tool will use it as a base line and will generate chunks for |
| * the incremental updates. |
| * <p/> |
| * The indexer types could be one of default, min or full. You can also specify list of comma-separated custom index |
| * creators. An index creator should be a regular Plexus component, see |
| * {@link org.apache.maven.index.creator.MinimalArtifactInfoIndexCreator} and |
| * {@link org.apache.maven.index.creator.JarFileContentsIndexCreator}. |
| */ |
| public class NexusIndexerCli |
| extends AbstractCli |
| { |
| // Command line options |
| |
| public static final char REPO = 'r'; |
| |
| public static final char INDEX = 'i'; |
| |
| public static final char NAME = 'n'; |
| |
| public static final char TYPE = 't'; |
| |
| public static final char TARGET_DIR = 'd'; |
| |
| public static final char CREATE_INCREMENTAL_CHUNKS = 'c'; |
| |
| public static final char CREATE_FILE_CHECKSUMS = 's'; |
| |
| public static final char INCREMENTAL_CHUNK_KEEP_COUNT = 'k'; |
| |
| public static final char UNPACK = 'u'; |
| |
| private static final long MB = 1024 * 1024; |
| |
| private Options options; |
| |
| private int status = 0; |
| |
| public static void main( String[] args ) |
| throws Exception |
| { |
| NexusIndexerCli cli = new NexusIndexerCli(); |
| |
| cli.execute( args ); |
| |
| System.exit( cli.status ); |
| } |
| |
| @Override |
| public int execute( String[] arg0, ClassWorld arg1 ) |
| { |
| int value = super.execute( arg0, arg1 ); |
| |
| if ( status == 0 ) |
| { |
| status = value; |
| } |
| |
| return status; |
| } |
| |
| @Override |
| public int execute( String[] args ) |
| { |
| int value = super.execute( args ); |
| |
| if ( status == 0 ) |
| { |
| status = value; |
| } |
| |
| return status; |
| } |
| |
| @Override |
| protected void showError( String message, Exception e, boolean show ) |
| { |
| status = 1; |
| super.showError( message, e, show ); |
| } |
| |
| @Override |
| protected int showFatalError( String message, Exception e, boolean show ) |
| { |
| status = 1; |
| return super.showFatalError( message, e, show ); |
| } |
| |
| @Override |
| public CommandLine parse( String[] args ) |
| throws ParseException |
| { |
| try |
| { |
| return super.parse( args ); |
| } |
| catch ( ParseException e ) |
| { |
| status = 1; |
| throw e; |
| } |
| } |
| |
| @Override |
| public String getPomPropertiesPath() |
| { |
| return "META-INF/maven/org.sonatype.nexus/nexus-indexer/pom.properties"; |
| } |
| |
| @Override |
| @SuppressWarnings( "static-access" ) |
| public Options buildCliOptions( Options options ) |
| { |
| this.options = options; |
| |
| options.addOption( OptionBuilder.withLongOpt( "index" ).hasArg() // |
| .withDescription( "Path to the index folder." ).create( INDEX ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "destination" ).hasArg() // |
| .withDescription( "Target folder." ).create( TARGET_DIR ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "repository" ).hasArg() // |
| .withDescription( "Path to the Maven repository." ).create( REPO ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "name" ).hasArg() // |
| .withDescription( "Repository name." ).create( NAME ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "chunks" ) // |
| .withDescription( "Create incremental chunks." ).create( CREATE_INCREMENTAL_CHUNKS ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "keep" ).hasArg().withDescription( |
| "Number of incremental chunks to keep." ).create( INCREMENTAL_CHUNK_KEEP_COUNT ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "checksums" ) // |
| .withDescription( "Create checksums for all files (sha1, md5)." ).create( CREATE_FILE_CHECKSUMS ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "type" ).hasArg() // |
| .withDescription( "Indexer type (default, min, full or comma separated list of custom types)." ).create( TYPE ) ); |
| |
| options.addOption( OptionBuilder.withLongOpt( "unpack" ) // |
| .withDescription( "Unpack an index file" ).create( UNPACK ) ); |
| |
| return options; |
| } |
| |
| @Override |
| public void displayHelp() |
| { |
| System.out.println(); |
| |
| HelpFormatter formatter = new HelpFormatter(); |
| |
| formatter.printHelp( "nexus-indexer [options]", "\nOptions:", options, "\n" ); |
| } |
| |
| public void displayHelp( String message ) |
| { |
| System.out.println(); |
| |
| System.out.println( message ); |
| |
| System.out.println(); |
| |
| displayHelp(); |
| } |
| |
| @Override |
| public void invokePlexusComponent( final CommandLine cli, PlexusContainer plexus ) |
| throws Exception |
| { |
| final DefaultContainerConfiguration configuration = new DefaultContainerConfiguration(); |
| configuration.setClassWorld( ( (DefaultPlexusContainer) plexus ).getClassWorld() ); |
| configuration.setClassPathScanning( PlexusConstants.SCANNING_INDEX ); |
| |
| // replace plexus, as PlexusCli is blunt, does not allow to modify configuration |
| // TODO: get rid of PlexusCli use! |
| plexus = new DefaultPlexusContainer( configuration ); |
| |
| if ( cli.hasOption( QUIET ) ) |
| { |
| setLogLevel( plexus, Logger.LEVEL_DISABLED ); |
| } |
| else if ( cli.hasOption( DEBUG ) ) |
| { |
| setLogLevel( plexus, Logger.LEVEL_DEBUG ); |
| } |
| else if ( cli.hasOption( ERRORS ) ) |
| { |
| setLogLevel( plexus, Logger.LEVEL_ERROR ); |
| } |
| |
| if ( cli.hasOption( UNPACK ) ) |
| { |
| unpack( cli, plexus ); |
| } |
| else if ( cli.hasOption( INDEX ) && cli.hasOption( REPO ) ) |
| { |
| index( cli, plexus ); |
| } |
| else |
| { |
| status = 1; |
| |
| displayHelp( "Use either unpack (\"" + UNPACK + "\") or index (\"" + INDEX + "\" and \"" + REPO |
| + "\") options, but none has been found!" ); |
| } |
| } |
| |
| private void setLogLevel( PlexusContainer plexus, int logLevel ) |
| throws ComponentLookupException |
| { |
| plexus.lookup( LoggerManager.class ).setThresholds( logLevel ); |
| } |
| |
| private void index( final CommandLine cli, PlexusContainer plexus ) |
| throws ComponentLookupException, IOException, UnsupportedExistingLuceneIndexException |
| { |
| String indexDirectoryName = cli.getOptionValue( INDEX ); |
| |
| File indexFolder = new File( indexDirectoryName ); |
| |
| String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." ); |
| |
| File outputFolder = new File( outputDirectoryName ); |
| |
| File repositoryFolder = new File( cli.getOptionValue( REPO ) ); |
| |
| String repositoryName = cli.getOptionValue( NAME, indexFolder.getName() ); |
| |
| List<IndexCreator> indexers = getIndexers( cli, plexus ); |
| |
| boolean createChecksums = cli.hasOption( CREATE_FILE_CHECKSUMS ); |
| |
| boolean createIncrementalChunks = cli.hasOption( CREATE_INCREMENTAL_CHUNKS ); |
| |
| boolean debug = cli.hasOption( DEBUG ); |
| |
| boolean quiet = cli.hasOption( QUIET ); |
| |
| Integer chunkCount = |
| cli.hasOption( INCREMENTAL_CHUNK_KEEP_COUNT ) ? Integer.parseInt( cli.getOptionValue( INCREMENTAL_CHUNK_KEEP_COUNT ) ) |
| : null; |
| |
| if ( !quiet ) |
| { |
| System.err.printf( "Repository Folder: %s\n", repositoryFolder.getAbsolutePath() ); |
| System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() ); |
| System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() ); |
| System.err.printf( "Repository name: %s\n", repositoryName ); |
| System.err.printf( "Indexers: %s\n", indexers.toString() ); |
| |
| if ( createChecksums ) |
| { |
| System.err.printf( "Will create checksum files for all published files (sha1, md5).\n" ); |
| } |
| else |
| { |
| System.err.printf( "Will not create checksum files.\n" ); |
| } |
| |
| if ( createIncrementalChunks ) |
| { |
| System.err.printf( "Will create incremental chunks for changes, along with baseline file.\n" ); |
| } |
| else |
| { |
| System.err.printf( "Will create baseline file.\n" ); |
| } |
| } |
| |
| NexusIndexer indexer = plexus.lookup( NexusIndexer.class ); |
| |
| long tstart = System.currentTimeMillis(); |
| |
| IndexingContext context = indexer.addIndexingContext( // |
| repositoryName, // context id |
| repositoryName, // repository id |
| repositoryFolder, // repository folder |
| indexFolder, // index folder |
| null, // repositoryUrl |
| null, // index update url |
| indexers ); |
| |
| try |
| { |
| IndexPacker packer = plexus.lookup( IndexPacker.class ); |
| |
| ArtifactScanningListener listener = new IndexerListener( context, debug, quiet ); |
| |
| indexer.scan( context, listener, true ); |
| |
| IndexSearcher indexSearcher = context.acquireIndexSearcher(); |
| |
| try |
| { |
| IndexPackingRequest request = |
| new IndexPackingRequest(context, indexSearcher.getIndexReader(), outputFolder); |
| |
| request.setCreateChecksumFiles(createChecksums); |
| |
| request.setCreateIncrementalChunks(createIncrementalChunks); |
| |
| request.setFormats(Arrays.asList(IndexFormat.FORMAT_V1)); |
| |
| if (chunkCount != null) |
| { |
| request.setMaxIndexChunks(chunkCount.intValue()); |
| } |
| |
| packIndex(packer, request, debug, quiet); |
| } |
| finally |
| { |
| context.releaseIndexSearcher(indexSearcher); |
| } |
| |
| if ( !quiet ) |
| { |
| printStats( tstart ); |
| } |
| } |
| finally |
| { |
| indexer.removeIndexingContext( context, false ); |
| } |
| } |
| |
| private void unpack( CommandLine cli, PlexusContainer plexus ) |
| throws ComponentLookupException, IOException |
| { |
| final String indexDirectoryName = cli.getOptionValue( INDEX, "." ); |
| final File indexFolder = new File( indexDirectoryName ).getCanonicalFile(); |
| final File indexArchive = new File( indexFolder, IndexingContext.INDEX_FILE_PREFIX + ".gz" ); |
| |
| final String outputDirectoryName = cli.getOptionValue( TARGET_DIR, "." ); |
| final File outputFolder = new File( outputDirectoryName ).getCanonicalFile(); |
| |
| final boolean quiet = cli.hasOption( QUIET ); |
| if ( !quiet ) |
| { |
| System.err.printf( "Index Folder: %s\n", indexFolder.getAbsolutePath() ); |
| System.err.printf( "Output Folder: %s\n", outputFolder.getAbsolutePath() ); |
| } |
| |
| long tstart = System.currentTimeMillis(); |
| |
| final List<IndexCreator> indexers = getIndexers( cli, plexus ); |
| |
| try (BufferedInputStream is = new BufferedInputStream( new FileInputStream( indexArchive ) ); // |
| FSDirectory directory = FSDirectory.open( outputFolder.toPath() )) |
| { |
| DefaultIndexUpdater.unpackIndexData( is, directory, (IndexingContext) Proxy.newProxyInstance( |
| getClass().getClassLoader(), new Class[] { IndexingContext.class }, new PartialImplementation() |
| { |
| public List<IndexCreator> getIndexCreators() |
| { |
| return indexers; |
| } |
| } ) |
| |
| ); |
| } |
| |
| if ( !quiet ) |
| { |
| printStats( tstart ); |
| } |
| } |
| |
| private List<IndexCreator> getIndexers( final CommandLine cli, PlexusContainer plexus ) |
| throws ComponentLookupException |
| { |
| String type = "default"; |
| |
| if ( cli.hasOption( TYPE ) ) |
| { |
| type = cli.getOptionValue( TYPE ); |
| } |
| |
| List<IndexCreator> indexers = new ArrayList<IndexCreator>(); // NexusIndexer.DEFAULT_INDEX; |
| |
| if ( "default".equals( type ) ) |
| { |
| indexers.add( plexus.lookup( IndexCreator.class, "min" ) ); |
| indexers.add( plexus.lookup( IndexCreator.class, "jarContent" ) ); |
| } |
| else if ( "full".equals( type ) ) |
| { |
| for ( Object component : plexus.lookupList( IndexCreator.class ) ) |
| { |
| indexers.add( (IndexCreator) component ); |
| } |
| } |
| else |
| { |
| for ( String hint : type.split( "," ) ) |
| { |
| indexers.add( plexus.lookup( IndexCreator.class, hint ) ); |
| } |
| } |
| return indexers; |
| } |
| |
| private void packIndex( IndexPacker packer, IndexPackingRequest request, boolean debug, boolean quiet ) |
| { |
| try |
| { |
| packer.packIndex( request ); |
| } |
| catch ( IOException e ) |
| { |
| if ( !quiet ) |
| { |
| System.err.printf( "Cannot zip index; \n", e.getMessage() ); |
| |
| if ( debug ) |
| { |
| e.printStackTrace(); |
| } |
| } |
| } |
| } |
| |
| private void printStats( final long startTimeInMillis ) |
| { |
| long t = System.currentTimeMillis() - startTimeInMillis; |
| |
| long s = t / 1000L; |
| if ( t > 60 * 1000 ) |
| { |
| long m = t / 1000L / 60L; |
| |
| System.err.printf( "Total time: %d min %d sec\n", m, s - ( m * 60 ) ); |
| } |
| else |
| { |
| System.err.printf( "Total time: %d sec\n", s ); |
| } |
| |
| Runtime r = Runtime.getRuntime(); |
| |
| System.err.printf( "Final memory: %dM/%dM\n", // |
| ( r.totalMemory() - r.freeMemory() ) / MB, r.totalMemory() / MB ); |
| } |
| |
| /** |
| * Scanner listener |
| */ |
| private static final class IndexerListener |
| implements ArtifactScanningListener |
| { |
| private final IndexingContext context; |
| |
| private final boolean debug; |
| |
| private boolean quiet; |
| |
| private long ts = System.currentTimeMillis(); |
| |
| private int count; |
| |
| IndexerListener( IndexingContext context, boolean debug, boolean quiet ) |
| { |
| this.context = context; |
| this.debug = debug; |
| this.quiet = quiet; |
| } |
| |
| public void scanningStarted( IndexingContext context ) |
| { |
| if ( !quiet ) |
| { |
| System.err.println( "Scanning started" ); |
| } |
| } |
| |
| public void artifactDiscovered( ArtifactContext ac ) |
| { |
| count++; |
| |
| long t = System.currentTimeMillis(); |
| |
| ArtifactInfo ai = ac.getArtifactInfo(); |
| |
| if ( !quiet && debug && "maven-plugin".equals( ai.getPackaging() ) ) |
| { |
| System.err.printf( "Plugin: %s:%s:%s - %s %s\n", // |
| ai.getGroupId(), ai.getArtifactId(), ai.getVersion(), ai.getPrefix(), "" + ai.getGoals() ); |
| } |
| |
| if ( !quiet && ( debug || ( t - ts ) > 2000L ) ) |
| { |
| System.err.printf( " %6d %s\n", count, formatFile( ac.getPom() ) ); |
| ts = t; |
| } |
| } |
| |
| public void artifactError( ArtifactContext ac, Exception e ) |
| { |
| if ( !quiet ) |
| { |
| System.err.printf( "! %6d %s - %s\n", count, formatFile( ac.getPom() ), e.getMessage() ); |
| |
| System.err.printf( " %s\n", formatFile( ac.getArtifact() ) ); |
| |
| if ( debug ) |
| { |
| e.printStackTrace(); |
| } |
| } |
| |
| ts = System.currentTimeMillis(); |
| } |
| |
| private String formatFile( File file ) |
| { |
| return file.getAbsolutePath().substring( context.getRepository().getAbsolutePath().length() + 1 ); |
| } |
| |
| public void scanningFinished( IndexingContext context, ScanningResult result ) |
| { |
| if ( !quiet ) |
| { |
| if ( result.hasExceptions() ) |
| { |
| System.err.printf( "Scanning errors: %s\n", result.getExceptions().size() ); |
| } |
| |
| System.err.printf( "Artifacts added: %s\n", result.getTotalFiles() ); |
| System.err.printf( "Artifacts deleted: %s\n", result.getDeletedFiles() ); |
| } |
| } |
| } |
| |
| } |