Remove DownloadMojo, use a file for the index (#2)
diff --git a/README.md b/README.md
index 8ecbdf2..994b838 100755
--- a/README.md
+++ b/README.md
@@ -33,5 +33,5 @@
### Reporting Bugs
-Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANS/issues/ using the component **mavenutilities**
+Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANSINFRA/issues/ using the component **MU - Apache NetBeans repository plugin**
diff --git a/pom.xml b/pom.xml
index 4013262..56d1dcb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,7 +38,7 @@
<url>https://bits.netbeans.org/mavenutilities/nb-repository-plugin</url>
<issueManagement>
<system>jira</system>
- <url>https://issues.apache.org/jira/browse/NETBEANS</url>
+ <url>https://issues.apache.org/jira/browse/NETBEANSINFRA</url>
</issueManagement>
<prerequisites>
<maven>3.1.1</maven>
@@ -55,7 +55,7 @@
<developerConnection>scm:git:https://gitbox.apache.org/repos/asf/netbeans-mavenutils-nb-repository-plugin.git</developerConnection>
<url>https://github.com/apache/netbeans-mavenutils-nb-repository-plugin</url>
<tag>HEAD</tag>
- </scm>
+ </scm>
<!-- TODO need to override parent version value... I suppose this will eventually end up in mojo parent pom, check regularly -->
<build>
@@ -104,7 +104,7 @@
<configuration>
<signature>
<groupId>org.codehaus.mojo.signature</groupId>
- <artifactId>java17</artifactId>
+ <artifactId>java18</artifactId>
<version>1.0</version>
</signature>
</configuration>
@@ -138,7 +138,8 @@
<!-- configure github milestone ? -->
<!--<onlyMilestoneIssues>false</onlyMilestoneIssues>
<onlyCurrentVersion>false</onlyCurrentVersion>-->
- <skip>true</skip>
+ <!--<skip>true</skip>-->
+ <component>12336704</component>
</configuration>
</plugin>
@@ -206,7 +207,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
<configuration>
- <targetJdk>1.6</targetJdk>
+ <targetJdk>1.8</targetJdk>
</configuration>
</plugin>
<plugin>
@@ -314,11 +315,6 @@
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
- <artifactId>maven-artifact</artifactId>
- <version>${maven.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>${maven.version}</version>
</dependency>
@@ -339,11 +335,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.lucene</groupId>
- <artifactId>lucene-core</artifactId>
- <version>5.5.5</version>
- </dependency>
- <dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>${maven.version}</version>
@@ -373,27 +364,10 @@
<type>jar</type>
</dependency>
<dependency>
- <groupId>org.apache.maven.indexer</groupId>
- <artifactId>indexer-core</artifactId>
- <version>6.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-http</artifactId>
- <version>${wagon.version}</version>
- <type>jar</type>
- </dependency>
- <dependency>
<groupId>${project.groupId}</groupId>
<artifactId>nb-shared</artifactId>
<version>1.3</version>
</dependency>
- <dependency>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-provider-api</artifactId>
- <version>${wagon.version}</version>
- <scope>compile</scope>
- </dependency>
</dependencies>
<properties>
<mojo.java.target>1.6</mojo.java.target>
diff --git a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java
deleted file mode 100644
index e2c9eff..0000000
--- a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java
+++ /dev/null
@@ -1,175 +0,0 @@
-package org.apache.netbeans.nbm.repository;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.File;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import org.apache.maven.artifact.manager.WagonManager;
-import org.apache.maven.index.NexusIndexer;
-import org.apache.maven.index.context.IndexCreator;
-import org.apache.maven.index.context.IndexingContext;
-import org.apache.maven.index.updater.IndexUpdateRequest;
-import org.apache.maven.index.updater.IndexUpdater;
-import org.apache.maven.index.updater.ResourceFetcher;
-import org.apache.maven.index.updater.WagonHelper;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.wagon.Wagon;
-import org.apache.maven.wagon.events.TransferEvent;
-import org.apache.maven.wagon.events.TransferListener;
-import org.apache.maven.wagon.providers.http.HttpWagon;
-import org.apache.maven.wagon.proxy.ProxyInfo;
-import org.codehaus.plexus.PlexusConstants;
-import org.codehaus.plexus.PlexusContainer;
-import org.codehaus.plexus.context.Context;
-import org.codehaus.plexus.context.ContextException;
-import org.codehaus.plexus.personality.plexus.lifecycle.phase.Contextualizable;
-
-/**
- * Goal for retrieving and expanding the lucene index of the given repository.
- * That in turn is used by the <code>populate</code> goal.
- * @author Milos Kleint
- */
-@Mojo( name = "download", aggregator = true, requiresProject = false )
-public class DownloadIndexMojo extends AbstractMojo implements Contextualizable
-{
-
- /**
- * url of the repository to download index from. Please note that if you already have
- * an existing index at <code>nexusIndexDirectory</code>, you should always use the same url for that directory.
- */
- @Parameter( required = true, property = "repositoryUrl" )
- private String repositoryUrl;
-
- /**
- * location on disk where the index should be created. either empty or with existing index from same repository.
- * Then only update check will be performed.
- */
- @Parameter( required = true, property = "nexusIndexDirectory" )
- private File nexusIndexDirectory;
-
- @Component
- IndexUpdater remoteIndexUpdater;
-
- @Component
- NexusIndexer indexer;
-
- PlexusContainer container;
-
-
- @Component
- WagonManager wagonManager;
-
- @Override
- public void execute() throws MojoExecutionException, MojoFailureException
- {
- try
- {
- List<IndexCreator> creators = new ArrayList<IndexCreator>();
- creators.addAll( container.lookupList( IndexCreator.class ) );
- String indexurl = repositoryUrl + ( !repositoryUrl.endsWith( "/" ) ? "/" : "" ) + ".index";
- IndexingContext indexingContext = indexer.addIndexingContextForced(
- "central", // context id
- "central", // repository id
- null, // repository folder
- nexusIndexDirectory,
- repositoryUrl, // repositoryUrl
- indexurl,
- creators );
-
- String protocol = URI.create( repositoryUrl ).getScheme();
- ProxyInfo wagonProxy = wagonManager.getProxy( protocol );
- TransferListener tr = new TransferListener()
- {
-
- @Override
- public void transferInitiated( TransferEvent transferEvent )
- {
- getLog().info( "Initiated connection to " + repositoryUrl );
- }
-
- @Override
- public void transferStarted( TransferEvent transferEvent )
- {
- getLog().info( "Started transfer of " + repositoryUrl + "/.index/"
- + transferEvent.getResource().toString() );
- }
-
- @Override
- public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
- {
- }
-
- @Override
- public void transferCompleted( TransferEvent transferEvent )
- {
- getLog().info( "Finished transfer of " + repositoryUrl + "/.index/"
- + transferEvent.getResource().toString() );
- }
-
- @Override
- public void transferError( TransferEvent transferEvent )
- {
- getLog().error( "Failed transfer of " + repositoryUrl + "/.index/"
- + transferEvent.getResource().toString(), transferEvent.getException() );
- }
-
- @Override
- public void debug( String message )
- {
- }
- };
- // MINDEXER-42: cannot use WagonHelper.getWagonResourceFetcher
- Wagon wagon = container.lookup( Wagon.class, protocol );
- if ( wagon instanceof HttpWagon )
- { //#216401
- HttpWagon httpwagon = (HttpWagon) wagon;
- //#215343
- Properties p = new Properties();
- p.setProperty( "User-Agent", "mojo/nb-repository-plugin" );
- httpwagon.setHttpHeaders( p );
- }
-
- ResourceFetcher fetcher = new WagonHelper.WagonFetcher( wagon, tr, null, wagonProxy );
- IndexUpdateRequest iur = new IndexUpdateRequest( indexingContext, fetcher );
-
- remoteIndexUpdater.fetchAndUpdateIndex( iur );
- indexer.removeIndexingContext( indexingContext, false );
- }
- catch ( Exception ex )
- {
- throw new MojoExecutionException( "Cannot download index", ex );
- }
- }
-
- @Override
- public void contextualize( Context context ) throws ContextException
- {
- this.container = ( PlexusContainer ) context.get( PlexusConstants.PLEXUS_KEY );
- }
-
-}
diff --git a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
index d033023..a84b20d 100644
--- a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
+++ b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
@@ -27,6 +27,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
+import java.nio.file.Files;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
@@ -43,17 +44,6 @@
import java.util.jar.Manifest;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.deployer.ArtifactDeployer;
import org.apache.maven.artifact.deployer.ArtifactDeploymentException;
@@ -88,7 +78,6 @@
import org.apache.tools.ant.taskdefs.PathConvert;
import org.apache.tools.ant.types.FileSet;
import org.codehaus.plexus.util.IOUtil;
-import org.codehaus.plexus.util.StringUtils;
/**
* A goal for identifying NetBeans modules from the installation and populating the local
@@ -191,17 +180,15 @@
protected String forcedVersion;
/**
- * When specified it points to a directory containing a Maven Indexer
- * (Nexus) Lucene index. This index will be used to find external libraries that
- * are referenced by NetBeans modules and populate the POM metadata with correct
- * dependencies. Any dependencies not found this way, will be generated with a unique
+ * When specified it points to a file containing a merge of all binaries-list
+ * sha1;coordinate;module
+ * Any dependencies not found this way, will be generated with a unique
* id under the org.netbeans.external groupId.
* <p/>
- * Use the {@code download} goal to retrieve the index.
- * @since 3.0
+ * @since 1.16
*/
- @Parameter( property = "nexusIndexDirectory" )
- private File nexusIndexDirectory;
+ @Parameter( property = "externallist" )
+ private File externallist;
/**
* Whether to create cluster POMs in the {@code org.netbeans.cluster} group.
@@ -323,22 +310,6 @@
"When skipping install to local repository, one shall define the deployUrl parameter" );
}
- IndexSearcher searcher = null;
- if ( nexusIndexDirectory != null && nexusIndexDirectory.exists() )
- {
- try
- {
- Directory nexusDir = FSDirectory.open( nexusIndexDirectory.toPath() );
- IndexReader nexusReader = DirectoryReader.open( nexusDir );
- searcher = new IndexSearcher( nexusReader );
- getLog().info( "Opened index with " + nexusReader.numDocs() + " documents" );
- }
- catch ( IOException ex )
- {
- getLog().error( "Could not open " + nexusIndexDirectory, ex );
- }
- }
-
if ( netbeansInstallDirectory == null )
{
Input input = (Input) antProject.createTask( "input" );
@@ -387,8 +358,8 @@
String prop = antProject.getProperty( "netbeansincludes" );
StringTokenizer tok = new StringTokenizer( prop, "," );
- HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<ModuleWrapper, Artifact>();
- HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<String, Collection<ModuleWrapper>>();
+ HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<>();
+ HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<>();
while ( tok.hasMoreTokens() )
{
String token = tok.nextToken();
@@ -420,7 +391,7 @@
Artifact art = createArtifact( artifact, version, group );
if ( examinator.isOsgiBundle() )
{
- Dependency dep = findExternal( searcher, module );
+ Dependency dep = findExternal( module );
if ( dep != null )
{
// XXX use those coords instead of publishing this
@@ -433,13 +404,13 @@
Collection<ModuleWrapper> col = clusters.get( clust );
if ( col == null )
{
- col = new ArrayList<ModuleWrapper>();
+ col = new ArrayList<>();
clusters.put( clust, col );
}
col.add( wr );
}
}
- List<ModuleWrapper> wrapperList = new ArrayList<ModuleWrapper>( moduleDefinitions.keySet() );
+ List<ModuleWrapper> wrapperList = new ArrayList<>( moduleDefinitions.keySet() );
int count = wrapperList.size() + 1;
int index = 0;
File javadocRoot = null;
@@ -477,7 +448,7 @@
}
}
- List<ExternalsWrapper> externals = new ArrayList<ExternalsWrapper>();
+ List<ExternalsWrapper> externals = new ArrayList<>();
try
{
for ( Map.Entry<ModuleWrapper, Artifact> elem : moduleDefinitions.entrySet() )
@@ -486,7 +457,7 @@
Artifact art = elem.getValue();
index = index + 1;
getLog().info( "Processing " + index + "/" + count );
- File pom = createMavenProject( man, wrapperList, externals, searcher );
+ File pom = createMavenProject( man, wrapperList, externals );
ArtifactMetadata metadata = new ProjectArtifactMetadata( art, pom );
art.addMetadata( metadata );
File javadoc = null;
@@ -784,7 +755,7 @@
}
private File createMavenProject( ModuleWrapper wrapper, List<ModuleWrapper> wrapperList,
- List<ExternalsWrapper> externalsList, IndexSearcher searcher )
+ List<ExternalsWrapper> externalsList )
throws MojoExecutionException
{
Model mavenModel = new Model();
@@ -799,7 +770,7 @@
mavenModel.setParent( artefactParent );
}
ExamineManifest man = wrapper.getModuleManifest();
- List<Dependency> deps = new ArrayList<Dependency>();
+ List<Dependency> deps = new ArrayList<>();
if ( !man.getDependencyTokens().isEmpty() )
{
for ( String elem : man.getDependencyTokens() )
@@ -899,7 +870,7 @@
File f = new File( wrapper.getFile().getParentFile(), path );
if ( f.exists() )
{
- Dependency dep = findExternal( searcher, f );
+ Dependency dep = findExternal( f );
if ( dep != null )
{
deps.add( dep );
@@ -963,61 +934,38 @@
return fil;
}
- private Dependency findExternal( IndexSearcher searcher, File f )
+ private Dependency findExternal( File f )
{
- if ( searcher == null )
+ if ( externallist == null )
{
return null;
}
try
{
+ List<String> content512 = Files.readAllLines( externallist.toPath() );
MessageDigest shaDig = MessageDigest.getInstance( "SHA1" );
- InputStream is = new FileInputStream( f );
- try
+
+ try ( InputStream is = new FileInputStream( f ); OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig ); )
{
- OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig );
IOUtil.copy( is, os );
- os.close();
}
- finally
+ String sha1 = encode ( shaDig.digest() ).toUpperCase();
+ for ( String string : content512 )
{
- is.close();
- }
- String sha = encode( shaDig.digest() );
- TermQuery q = new TermQuery( new Term( "1", sha ) );
- TopScoreDocCollector collector = TopScoreDocCollector.create( 5 );
- searcher.search( q, collector );
- ScoreDoc[] hits = collector.topDocs().scoreDocs;
- if ( hits.length >= 1 )
- {
- int docId = hits[0].doc;
- Document doc = searcher.doc( docId );
- IndexableField idField = doc.getField( "u" );
- if ( idField != null )
+ String[] split = string.split( ";" );
+ if ( split[0].equals( sha1 ) && split[1].contains( ":" ) )
{
- String id = idField.stringValue();
- String[] splits = StringUtils.split( id, "|" );
+ String[] splits = split[1].split( ":" );
Dependency dep = new Dependency();
dep.setArtifactId( splits[1] );
dep.setGroupId( splits[0] );
dep.setVersion( splits[2] );
dep.setType( "jar" );
- if ( splits.length > 3 && !"NA".equals( splits[3] ) )
- {
- dep.setClassifier( splits[3] );
- }
getLog().info( "found match " + splits[0] + ":" + splits[1] + ":" + splits[2] + " for " + f.getName() );
return dep;
- }
- else
- {
- getLog().error( "no idField for " + q );
- }
+ }
}
- else
- {
- getLog().info( "no repository match for " + f.getName() );
- }
+ getLog().info( "no repository match for " + f.getName() + f.getAbsolutePath() + " with sha " + sha1 );
}
catch ( Exception x )
{
@@ -1089,7 +1037,7 @@
{
mavenModel.setParent( artefactParent );
}
- List<Dependency> deps = new ArrayList<Dependency>();
+ List<Dependency> deps = new ArrayList<>();
for ( ModuleWrapper wr : mods )
{
Dependency dep = new Dependency();
@@ -1227,12 +1175,12 @@
List<Dependency> deps;
- public ModuleWrapper( String module )
+ ModuleWrapper( String module )
{
this.module = module;
}
- public ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil )
+ ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil )
{
man = manifest;
artifact = art;
