Remove DownloadMojo, use a file for the index (#2)

diff --git a/README.md b/README.md
index 8ecbdf2..994b838 100755
--- a/README.md
+++ b/README.md
@@ -33,5 +33,5 @@
 
 ### Reporting Bugs
 
-Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANS/issues/ using the component **mavenutilities**
+Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANSINFRA/issues/ using the component **MU - Apache NetBeans repository plugin**
 
diff --git a/pom.xml b/pom.xml
index 4013262..56d1dcb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,7 +38,7 @@
     <url>https://bits.netbeans.org/mavenutilities/nb-repository-plugin</url>
     <issueManagement>
         <system>jira</system>
-        <url>https://issues.apache.org/jira/browse/NETBEANS</url>
+        <url>https://issues.apache.org/jira/browse/NETBEANSINFRA</url>
     </issueManagement>
     <prerequisites>
         <maven>3.1.1</maven>
@@ -55,7 +55,7 @@
         <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/netbeans-mavenutils-nb-repository-plugin.git</developerConnection>
         <url>https://github.com/apache/netbeans-mavenutils-nb-repository-plugin</url>
       <tag>HEAD</tag>
-  </scm>
+    </scm>
     <!-- TODO need to override parent version value... I suppose this will eventually end up in mojo parent pom, check regularly -->    
 
     <build>
@@ -104,7 +104,7 @@
                         <configuration>
                             <signature>
                                 <groupId>org.codehaus.mojo.signature</groupId>
-                                <artifactId>java17</artifactId>
+                                <artifactId>java18</artifactId>
                                 <version>1.0</version>
                             </signature>
                         </configuration>
@@ -138,7 +138,8 @@
                     <!-- configure github milestone ? -->
                     <!--<onlyMilestoneIssues>false</onlyMilestoneIssues>
                     <onlyCurrentVersion>false</onlyCurrentVersion>-->
-                    <skip>true</skip>
+                    <!--<skip>true</skip>-->
+                    <component>12336704</component>
                 </configuration>
                
             </plugin>
@@ -206,7 +207,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-pmd-plugin</artifactId>
                <configuration>
-                    <targetJdk>1.6</targetJdk>
+                    <targetJdk>1.8</targetJdk>
                 </configuration>
             </plugin>
             <plugin>
@@ -314,11 +315,6 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.maven</groupId>
-            <artifactId>maven-artifact</artifactId>
-            <version>${maven.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.maven</groupId>
             <artifactId>maven-plugin-api</artifactId>
             <version>${maven.version}</version>
         </dependency>
@@ -339,11 +335,6 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.apache.lucene</groupId>
-            <artifactId>lucene-core</artifactId>
-            <version>5.5.5</version>
-        </dependency>
-        <dependency>
             <groupId>org.apache.maven</groupId>
             <artifactId>maven-model</artifactId>
             <version>${maven.version}</version>
@@ -373,27 +364,10 @@
             <type>jar</type>
         </dependency>
         <dependency>
-            <groupId>org.apache.maven.indexer</groupId>
-            <artifactId>indexer-core</artifactId>
-            <version>6.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.maven.wagon</groupId>
-            <artifactId>wagon-http</artifactId>
-            <version>${wagon.version}</version>
-            <type>jar</type>
-        </dependency>
-        <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>nb-shared</artifactId>
             <version>1.3</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.maven.wagon</groupId>
-            <artifactId>wagon-provider-api</artifactId>
-            <version>${wagon.version}</version>
-            <scope>compile</scope>
-        </dependency>
     </dependencies>
     <properties>
         <mojo.java.target>1.6</mojo.java.target>
diff --git a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java
deleted file mode 100644
index e2c9eff..0000000
--- a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java
+++ /dev/null
@@ -1,175 +0,0 @@
-package org.apache.netbeans.nbm.repository;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.File;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import org.apache.maven.artifact.manager.WagonManager;
-import org.apache.maven.index.NexusIndexer;
-import org.apache.maven.index.context.IndexCreator;
-import org.apache.maven.index.context.IndexingContext;
-import org.apache.maven.index.updater.IndexUpdateRequest;
-import org.apache.maven.index.updater.IndexUpdater;
-import org.apache.maven.index.updater.ResourceFetcher;
-import org.apache.maven.index.updater.WagonHelper;
-import org.apache.maven.plugin.AbstractMojo;
-import org.apache.maven.plugin.MojoExecutionException;
-import org.apache.maven.plugin.MojoFailureException;
-import org.apache.maven.plugins.annotations.Component;
-import org.apache.maven.plugins.annotations.Mojo;
-import org.apache.maven.plugins.annotations.Parameter;
-import org.apache.maven.wagon.Wagon;
-import org.apache.maven.wagon.events.TransferEvent;
-import org.apache.maven.wagon.events.TransferListener;
-import org.apache.maven.wagon.providers.http.HttpWagon;
-import org.apache.maven.wagon.proxy.ProxyInfo;
-import org.codehaus.plexus.PlexusConstants;
-import org.codehaus.plexus.PlexusContainer;
-import org.codehaus.plexus.context.Context;
-import org.codehaus.plexus.context.ContextException;
-import org.codehaus.plexus.personality.plexus.lifecycle.phase.Contextualizable;
-
-/**
- * Goal for retrieving and expanding the lucene index of the given repository. 
- * That in turn is used by the <code>populate</code> goal.
- * @author Milos Kleint
- */
-@Mojo( name = "download", aggregator = true, requiresProject = false )
-public class DownloadIndexMojo extends AbstractMojo implements Contextualizable 
-{
-
-    /**
-     * url of the repository to download index from. Please note that if you already have
-     * an existing index at <code>nexusIndexDirectory</code>, you should always use the same url for that directory.
-     */
-    @Parameter( required = true, property = "repositoryUrl" )
-    private String repositoryUrl;
-
-    /**
-     * location on disk where the index should be created. either empty or with existing index from same repository. 
-     * Then only update check will be performed.
-     */
-    @Parameter( required = true, property = "nexusIndexDirectory" )
-    private File nexusIndexDirectory;
- 
-    @Component
-    IndexUpdater remoteIndexUpdater;
-
-    @Component
-    NexusIndexer indexer;
-
-    PlexusContainer container;
-
-
-    @Component
-    WagonManager wagonManager;
-
-    @Override
-    public void execute() throws MojoExecutionException, MojoFailureException
-    {
-        try
-        {
-            List<IndexCreator> creators = new ArrayList<IndexCreator>();
-            creators.addAll( container.lookupList( IndexCreator.class ) );
-            String indexurl = repositoryUrl + ( !repositoryUrl.endsWith( "/" ) ? "/" : "" ) + ".index";
-            IndexingContext indexingContext = indexer.addIndexingContextForced(
-                                    "central", // context id
-                                    "central", // repository id
-                                    null, // repository folder
-                                    nexusIndexDirectory,
-                                    repositoryUrl, // repositoryUrl
-                                    indexurl,
-                                    creators );
-
-            String protocol = URI.create( repositoryUrl ).getScheme();
-            ProxyInfo wagonProxy = wagonManager.getProxy( protocol );
-            TransferListener tr = new TransferListener() 
-            {
-
-                @Override
-                public void transferInitiated( TransferEvent transferEvent )
-                {
-                    getLog().info( "Initiated connection to " + repositoryUrl );
-                }
-
-                @Override
-                public void transferStarted( TransferEvent transferEvent )
-                {
-                    getLog().info( "Started transfer of " + repositoryUrl + "/.index/" 
-                            + transferEvent.getResource().toString() );
-                }
-
-                @Override
-                public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length )
-                {
-                }
-
-                @Override
-                public void transferCompleted( TransferEvent transferEvent )
-                {
-                    getLog().info( "Finished transfer of " + repositoryUrl + "/.index/" 
-                            + transferEvent.getResource().toString() );
-                }
-
-                @Override
-                public void transferError( TransferEvent transferEvent )
-                {
-                    getLog().error( "Failed transfer of " + repositoryUrl + "/.index/" 
-                            + transferEvent.getResource().toString(), transferEvent.getException() );
-                }
-
-                @Override
-                public void debug( String message )
-                {
-                }
-            };
-            // MINDEXER-42: cannot use WagonHelper.getWagonResourceFetcher
-            Wagon wagon = container.lookup( Wagon.class, protocol );
-            if ( wagon instanceof HttpWagon ) 
-            { //#216401
-                HttpWagon httpwagon = (HttpWagon) wagon;
-                //#215343
-                Properties p = new Properties();
-                p.setProperty( "User-Agent", "mojo/nb-repository-plugin" );
-                httpwagon.setHttpHeaders( p );
-            }
-
-            ResourceFetcher fetcher = new WagonHelper.WagonFetcher( wagon, tr, null, wagonProxy );
-            IndexUpdateRequest iur = new IndexUpdateRequest( indexingContext, fetcher );
-
-            remoteIndexUpdater.fetchAndUpdateIndex( iur );
-            indexer.removeIndexingContext( indexingContext, false );
-        }
-        catch ( Exception ex )
-        {
-            throw new MojoExecutionException( "Cannot download index", ex );
-        }
-    }
-
-    @Override
-    public void contextualize( Context context ) throws ContextException
-    {
-        this.container = ( PlexusContainer ) context.get( PlexusConstants.PLEXUS_KEY );
-    }
-
-}
diff --git a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
index d033023..a84b20d 100644
--- a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
+++ b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java
@@ -27,6 +27,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.math.BigInteger;
+import java.nio.file.Files;
 import java.security.DigestOutputStream;
 import java.security.MessageDigest;
 import java.util.ArrayList;
@@ -43,17 +44,6 @@
 import java.util.jar.Manifest;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopScoreDocCollector;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.artifact.deployer.ArtifactDeployer;
 import org.apache.maven.artifact.deployer.ArtifactDeploymentException;
@@ -88,7 +78,6 @@
 import org.apache.tools.ant.taskdefs.PathConvert;
 import org.apache.tools.ant.types.FileSet;
 import org.codehaus.plexus.util.IOUtil;
-import org.codehaus.plexus.util.StringUtils;
 
 /**
  * A goal for identifying NetBeans modules from the installation and populating the local
@@ -191,17 +180,15 @@
     protected String forcedVersion;
 
     /**
-     * When specified it points to a directory containing a Maven Indexer
-     * (Nexus) Lucene index. This index will be used to find external libraries that
-     * are referenced by NetBeans modules and populate the POM metadata with correct
-     * dependencies. Any dependencies not found this way, will be generated with a unique
+     * When specified it points to a file containing a merge of all binaries-list
+     * sha1;coordinate;module
+     * Any dependencies not found this way, will be generated with a unique
      * id under the org.netbeans.external groupId.
      * <p/>
-     * Use the {@code download} goal to retrieve the index.
-     * @since 3.0
+     * @since 1.16
      */
-    @Parameter( property = "nexusIndexDirectory" )
-    private File nexusIndexDirectory;
+    @Parameter( property = "externallist" )
+    private File externallist;
 
     /**
      * Whether to create cluster POMs in the {@code org.netbeans.cluster} group.
@@ -323,22 +310,6 @@
                     "When skipping install to local repository, one shall define the deployUrl parameter" );
         }
 
-        IndexSearcher searcher = null;
-        if ( nexusIndexDirectory != null && nexusIndexDirectory.exists() )
-        {
-            try
-            {
-                Directory nexusDir = FSDirectory.open( nexusIndexDirectory.toPath() );
-                IndexReader nexusReader = DirectoryReader.open( nexusDir );
-                searcher = new IndexSearcher( nexusReader );
-                getLog().info( "Opened index with " + nexusReader.numDocs() + " documents" );
-            }
-            catch ( IOException ex )
-            {
-                getLog().error( "Could not open " + nexusIndexDirectory, ex );
-            }
-        }
-
         if ( netbeansInstallDirectory == null )
         {
             Input input = (Input) antProject.createTask( "input" );
@@ -387,8 +358,8 @@
 
         String prop = antProject.getProperty( "netbeansincludes" );
         StringTokenizer tok = new StringTokenizer( prop, "," );
-        HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<ModuleWrapper, Artifact>();
-        HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<String, Collection<ModuleWrapper>>();
+        HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<>();
+        HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<>();
         while ( tok.hasMoreTokens() )
         {
             String token = tok.nextToken();
@@ -420,7 +391,7 @@
                 Artifact art = createArtifact( artifact, version, group );
                 if ( examinator.isOsgiBundle() )
                 {
-                    Dependency dep = findExternal( searcher, module );
+                    Dependency dep = findExternal( module );
                     if ( dep != null )
                     {
                         // XXX use those coords instead of publishing this
@@ -433,13 +404,13 @@
                 Collection<ModuleWrapper> col = clusters.get( clust );
                 if ( col == null )
                 {
-                    col = new ArrayList<ModuleWrapper>();
+                    col = new ArrayList<>();
                     clusters.put( clust, col );
                 }
                 col.add( wr );
             }
         }
-        List<ModuleWrapper> wrapperList = new ArrayList<ModuleWrapper>( moduleDefinitions.keySet() );
+        List<ModuleWrapper> wrapperList = new ArrayList<>( moduleDefinitions.keySet() );
         int count = wrapperList.size() + 1;
         int index = 0;
         File javadocRoot = null;
@@ -477,7 +448,7 @@
             }
         }
 
-        List<ExternalsWrapper> externals = new ArrayList<ExternalsWrapper>();
+        List<ExternalsWrapper> externals = new ArrayList<>();
         try
         {
             for ( Map.Entry<ModuleWrapper, Artifact> elem : moduleDefinitions.entrySet() )
@@ -486,7 +457,7 @@
                 Artifact art = elem.getValue();
                 index = index + 1;
                 getLog().info( "Processing " + index + "/" + count );
-                File pom = createMavenProject( man, wrapperList, externals, searcher );
+                File pom = createMavenProject( man, wrapperList, externals );
                 ArtifactMetadata metadata = new ProjectArtifactMetadata( art, pom );
                 art.addMetadata( metadata );
                 File javadoc = null;
@@ -784,7 +755,7 @@
     }
 
     private File createMavenProject( ModuleWrapper wrapper, List<ModuleWrapper> wrapperList,
-                                     List<ExternalsWrapper> externalsList, IndexSearcher searcher )
+                                     List<ExternalsWrapper> externalsList )
             throws MojoExecutionException
     {
         Model mavenModel = new Model();
@@ -799,7 +770,7 @@
             mavenModel.setParent( artefactParent );
         }
         ExamineManifest man = wrapper.getModuleManifest();
-        List<Dependency> deps = new ArrayList<Dependency>();
+        List<Dependency> deps = new ArrayList<>();
         if ( !man.getDependencyTokens().isEmpty() )
         {
             for ( String elem : man.getDependencyTokens() )
@@ -899,7 +870,7 @@
                 File f = new File( wrapper.getFile().getParentFile(), path );
                 if ( f.exists() )
                 {
-                    Dependency dep = findExternal( searcher, f );
+                    Dependency dep = findExternal( f );
                     if ( dep != null )
                     {
                         deps.add( dep );
@@ -963,61 +934,38 @@
         return fil;
     }
 
-    private Dependency findExternal( IndexSearcher searcher, File f )
+    private Dependency findExternal( File f )
     {
-        if ( searcher == null )
+        if ( externallist == null )
         {
             return null;
         }
         try
         {
+            List<String> content512 = Files.readAllLines( externallist.toPath() );
             MessageDigest shaDig = MessageDigest.getInstance( "SHA1" );
-            InputStream is = new FileInputStream( f );
-            try
+            
+            try ( InputStream is = new FileInputStream( f ); OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig ); )
             {
-                OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig );
                 IOUtil.copy( is, os );
-                os.close();
             }
-            finally
+            String sha1 = encode ( shaDig.digest() ).toUpperCase();
+            for ( String string : content512 ) 
             {
-                is.close();
-            }
-            String sha = encode( shaDig.digest() );
-            TermQuery q = new TermQuery( new Term( "1", sha ) );
-            TopScoreDocCollector collector = TopScoreDocCollector.create( 5 );
-            searcher.search( q, collector );
-            ScoreDoc[] hits = collector.topDocs().scoreDocs;
-            if ( hits.length >= 1 )
-            {
-                int docId = hits[0].doc;
-                Document doc = searcher.doc( docId );
-                IndexableField idField = doc.getField( "u" );
-                if ( idField != null )
+                String[] split = string.split( ";" );
+                if ( split[0].equals( sha1 ) && split[1].contains( ":" ) ) 
                 {
-                    String id = idField.stringValue();
-                    String[] splits = StringUtils.split( id, "|" );
+                    String[] splits = split[1].split( ":" );
                     Dependency dep = new Dependency();
                     dep.setArtifactId( splits[1] );
                     dep.setGroupId( splits[0] );
                     dep.setVersion( splits[2] );
                     dep.setType( "jar" );
-                    if ( splits.length > 3 && !"NA".equals( splits[3] ) )
-                    {
-                        dep.setClassifier( splits[3] );
-                    }
                     getLog().info( "found match " + splits[0] + ":" + splits[1] + ":" + splits[2] + " for " + f.getName() );
                     return dep;
-                }
-                else
-                {
-                    getLog().error( "no idField for " + q );
-                }
+                }            
             }
-            else
-            {
-                getLog().info( "no repository match for " + f.getName() );
-            }
+            getLog().info( "no repository match for " + f.getName() + f.getAbsolutePath() + " with sha " + sha1 );            
         }
         catch ( Exception x )
         {
@@ -1089,7 +1037,7 @@
         {
             mavenModel.setParent( artefactParent );
         }
-        List<Dependency> deps = new ArrayList<Dependency>();
+        List<Dependency> deps = new ArrayList<>();
         for ( ModuleWrapper wr : mods )
         {
             Dependency dep = new Dependency();
@@ -1227,12 +1175,12 @@
 
         List<Dependency> deps;
 
-        public ModuleWrapper( String module )
+        ModuleWrapper( String module )
         {
             this.module = module;
         }
 
-        public ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil )
+        ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil )
         {
             man = manifest;
             artifact = art;