This is an automated email from the ASF dual-hosted git repository. skygo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/netbeans-mavenutils-nb-repository-plugin.git
The following commit(s) were added to refs/heads/master by this push: new aa229c4 Remove DownloadMojo, use a file for the index (#2) aa229c4 is described below commit aa229c4d4f4250ff1846882ec33ffd154d47b924 Author: Eric Barboni <sk...@apache.org> AuthorDate: Sat Mar 7 23:27:59 2020 +0100 Remove DownloadMojo, use a file for the index (#2) --- README.md | 2 +- pom.xml | 38 +---- .../netbeans/nbm/repository/DownloadIndexMojo.java | 175 --------------------- .../nbm/repository/PopulateRepositoryMojo.java | 116 ++++---------- 4 files changed, 39 insertions(+), 292 deletions(-) diff --git a/README.md b/README.md index 8ecbdf2..994b838 100755 --- a/README.md +++ b/README.md @@ -33,5 +33,5 @@ ### Reporting Bugs -Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANS/issues/ using the component **mavenutilities** +Bugs should be reported to https://issues.apache.org/jira/projects/NETBEANSINFRA/issues/ using the component **MU - Apache NetBeans repository plugin** diff --git a/pom.xml b/pom.xml index 4013262..56d1dcb 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ under the License. <url>https://bits.netbeans.org/mavenutilities/nb-repository-plugin</url> <issueManagement> <system>jira</system> - <url>https://issues.apache.org/jira/browse/NETBEANS</url> + <url>https://issues.apache.org/jira/browse/NETBEANSINFRA</url> </issueManagement> <prerequisites> <maven>3.1.1</maven> @@ -55,7 +55,7 @@ under the License. <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/netbeans-mavenutils-nb-repository-plugin.git</developerConnection> <url>https://github.com/apache/netbeans-mavenutils-nb-repository-plugin</url> <tag>HEAD</tag> - </scm> + </scm> <!-- TODO need to override parent version value... I suppose this will eventually end up in mojo parent pom, check regularly --> <build> @@ -104,7 +104,7 @@ under the License. <configuration> <signature> <groupId>org.codehaus.mojo.signature</groupId> - <artifactId>java17</artifactId> + <artifactId>java18</artifactId> <version>1.0</version> </signature> </configuration> @@ -138,7 +138,8 @@ under the License. <!-- configure github milestone ? --> <!--<onlyMilestoneIssues>false</onlyMilestoneIssues> <onlyCurrentVersion>false</onlyCurrentVersion>--> - <skip>true</skip> + <!--<skip>true</skip>--> + <component>12336704</component> </configuration> </plugin> @@ -206,7 +207,7 @@ under the License. <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-pmd-plugin</artifactId> <configuration> - <targetJdk>1.6</targetJdk> + <targetJdk>1.8</targetJdk> </configuration> </plugin> <plugin> @@ -314,11 +315,6 @@ under the License. <dependencies> <dependency> <groupId>org.apache.maven</groupId> - <artifactId>maven-artifact</artifactId> - <version>${maven.version}</version> - </dependency> - <dependency> - <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> <version>${maven.version}</version> </dependency> @@ -339,11 +335,6 @@ under the License. <scope>test</scope> </dependency> <dependency> - <groupId>org.apache.lucene</groupId> - <artifactId>lucene-core</artifactId> - <version>5.5.5</version> - </dependency> - <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-model</artifactId> <version>${maven.version}</version> @@ -373,27 +364,10 @@ under the License. <type>jar</type> </dependency> <dependency> - <groupId>org.apache.maven.indexer</groupId> - <artifactId>indexer-core</artifactId> - <version>6.0.0</version> - </dependency> - <dependency> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-http</artifactId> - <version>${wagon.version}</version> - <type>jar</type> - </dependency> - <dependency> <groupId>${project.groupId}</groupId> <artifactId>nb-shared</artifactId> <version>1.3</version> </dependency> - <dependency> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-provider-api</artifactId> - <version>${wagon.version}</version> - <scope>compile</scope> - </dependency> </dependencies> <properties> <mojo.java.target>1.6</mojo.java.target> diff --git a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java deleted file mode 100644 index e2c9eff..0000000 --- a/src/main/java/org/apache/netbeans/nbm/repository/DownloadIndexMojo.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.apache.netbeans.nbm.repository; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.File; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import org.apache.maven.artifact.manager.WagonManager; -import org.apache.maven.index.NexusIndexer; -import org.apache.maven.index.context.IndexCreator; -import org.apache.maven.index.context.IndexingContext; -import org.apache.maven.index.updater.IndexUpdateRequest; -import org.apache.maven.index.updater.IndexUpdater; -import org.apache.maven.index.updater.ResourceFetcher; -import org.apache.maven.index.updater.WagonHelper; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.MojoFailureException; -import org.apache.maven.plugins.annotations.Component; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; -import org.apache.maven.wagon.Wagon; -import org.apache.maven.wagon.events.TransferEvent; -import org.apache.maven.wagon.events.TransferListener; -import org.apache.maven.wagon.providers.http.HttpWagon; -import org.apache.maven.wagon.proxy.ProxyInfo; -import org.codehaus.plexus.PlexusConstants; -import org.codehaus.plexus.PlexusContainer; -import org.codehaus.plexus.context.Context; -import org.codehaus.plexus.context.ContextException; -import org.codehaus.plexus.personality.plexus.lifecycle.phase.Contextualizable; - -/** - * Goal for retrieving and expanding the lucene index of the given repository. - * That in turn is used by the <code>populate</code> goal. - * @author Milos Kleint - */ -@Mojo( name = "download", aggregator = true, requiresProject = false ) -public class DownloadIndexMojo extends AbstractMojo implements Contextualizable -{ - - /** - * url of the repository to download index from. Please note that if you already have - * an existing index at <code>nexusIndexDirectory</code>, you should always use the same url for that directory. - */ - @Parameter( required = true, property = "repositoryUrl" ) - private String repositoryUrl; - - /** - * location on disk where the index should be created. either empty or with existing index from same repository. - * Then only update check will be performed. - */ - @Parameter( required = true, property = "nexusIndexDirectory" ) - private File nexusIndexDirectory; - - @Component - IndexUpdater remoteIndexUpdater; - - @Component - NexusIndexer indexer; - - PlexusContainer container; - - - @Component - WagonManager wagonManager; - - @Override - public void execute() throws MojoExecutionException, MojoFailureException - { - try - { - List<IndexCreator> creators = new ArrayList<IndexCreator>(); - creators.addAll( container.lookupList( IndexCreator.class ) ); - String indexurl = repositoryUrl + ( !repositoryUrl.endsWith( "/" ) ? "/" : "" ) + ".index"; - IndexingContext indexingContext = indexer.addIndexingContextForced( - "central", // context id - "central", // repository id - null, // repository folder - nexusIndexDirectory, - repositoryUrl, // repositoryUrl - indexurl, - creators ); - - String protocol = URI.create( repositoryUrl ).getScheme(); - ProxyInfo wagonProxy = wagonManager.getProxy( protocol ); - TransferListener tr = new TransferListener() - { - - @Override - public void transferInitiated( TransferEvent transferEvent ) - { - getLog().info( "Initiated connection to " + repositoryUrl ); - } - - @Override - public void transferStarted( TransferEvent transferEvent ) - { - getLog().info( "Started transfer of " + repositoryUrl + "/.index/" - + transferEvent.getResource().toString() ); - } - - @Override - public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length ) - { - } - - @Override - public void transferCompleted( TransferEvent transferEvent ) - { - getLog().info( "Finished transfer of " + repositoryUrl + "/.index/" - + transferEvent.getResource().toString() ); - } - - @Override - public void transferError( TransferEvent transferEvent ) - { - getLog().error( "Failed transfer of " + repositoryUrl + "/.index/" - + transferEvent.getResource().toString(), transferEvent.getException() ); - } - - @Override - public void debug( String message ) - { - } - }; - // MINDEXER-42: cannot use WagonHelper.getWagonResourceFetcher - Wagon wagon = container.lookup( Wagon.class, protocol ); - if ( wagon instanceof HttpWagon ) - { //#216401 - HttpWagon httpwagon = (HttpWagon) wagon; - //#215343 - Properties p = new Properties(); - p.setProperty( "User-Agent", "mojo/nb-repository-plugin" ); - httpwagon.setHttpHeaders( p ); - } - - ResourceFetcher fetcher = new WagonHelper.WagonFetcher( wagon, tr, null, wagonProxy ); - IndexUpdateRequest iur = new IndexUpdateRequest( indexingContext, fetcher ); - - remoteIndexUpdater.fetchAndUpdateIndex( iur ); - indexer.removeIndexingContext( indexingContext, false ); - } - catch ( Exception ex ) - { - throw new MojoExecutionException( "Cannot download index", ex ); - } - } - - @Override - public void contextualize( Context context ) throws ContextException - { - this.container = ( PlexusContainer ) context.get( PlexusConstants.PLEXUS_KEY ); - } - -} diff --git a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java index d033023..a84b20d 100644 --- a/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java +++ b/src/main/java/org/apache/netbeans/nbm/repository/PopulateRepositoryMojo.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.math.BigInteger; +import java.nio.file.Files; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.ArrayList; @@ -43,17 +44,6 @@ import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.apache.maven.artifact.Artifact; import org.apache.maven.artifact.deployer.ArtifactDeployer; import org.apache.maven.artifact.deployer.ArtifactDeploymentException; @@ -88,7 +78,6 @@ import org.apache.tools.ant.taskdefs.Input; import org.apache.tools.ant.taskdefs.PathConvert; import org.apache.tools.ant.types.FileSet; import org.codehaus.plexus.util.IOUtil; -import org.codehaus.plexus.util.StringUtils; /** * A goal for identifying NetBeans modules from the installation and populating the local @@ -191,17 +180,15 @@ public class PopulateRepositoryMojo protected String forcedVersion; /** - * When specified it points to a directory containing a Maven Indexer - * (Nexus) Lucene index. This index will be used to find external libraries that - * are referenced by NetBeans modules and populate the POM metadata with correct - * dependencies. Any dependencies not found this way, will be generated with a unique + * When specified it points to a file containing a merge of all binaries-list + * sha1;coordinate;module + * Any dependencies not found this way, will be generated with a unique * id under the org.netbeans.external groupId. * <p/> - * Use the {@code download} goal to retrieve the index. - * @since 3.0 + * @since 1.16 */ - @Parameter( property = "nexusIndexDirectory" ) - private File nexusIndexDirectory; + @Parameter( property = "externallist" ) + private File externallist; /** * Whether to create cluster POMs in the {@code org.netbeans.cluster} group. @@ -323,22 +310,6 @@ public class PopulateRepositoryMojo "When skipping install to local repository, one shall define the deployUrl parameter" ); } - IndexSearcher searcher = null; - if ( nexusIndexDirectory != null && nexusIndexDirectory.exists() ) - { - try - { - Directory nexusDir = FSDirectory.open( nexusIndexDirectory.toPath() ); - IndexReader nexusReader = DirectoryReader.open( nexusDir ); - searcher = new IndexSearcher( nexusReader ); - getLog().info( "Opened index with " + nexusReader.numDocs() + " documents" ); - } - catch ( IOException ex ) - { - getLog().error( "Could not open " + nexusIndexDirectory, ex ); - } - } - if ( netbeansInstallDirectory == null ) { Input input = (Input) antProject.createTask( "input" ); @@ -387,8 +358,8 @@ public class PopulateRepositoryMojo String prop = antProject.getProperty( "netbeansincludes" ); StringTokenizer tok = new StringTokenizer( prop, "," ); - HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<ModuleWrapper, Artifact>(); - HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<String, Collection<ModuleWrapper>>(); + HashMap<ModuleWrapper, Artifact> moduleDefinitions = new HashMap<>(); + HashMap<String, Collection<ModuleWrapper>> clusters = new HashMap<>(); while ( tok.hasMoreTokens() ) { String token = tok.nextToken(); @@ -420,7 +391,7 @@ public class PopulateRepositoryMojo Artifact art = createArtifact( artifact, version, group ); if ( examinator.isOsgiBundle() ) { - Dependency dep = findExternal( searcher, module ); + Dependency dep = findExternal( module ); if ( dep != null ) { // XXX use those coords instead of publishing this @@ -433,13 +404,13 @@ public class PopulateRepositoryMojo Collection<ModuleWrapper> col = clusters.get( clust ); if ( col == null ) { - col = new ArrayList<ModuleWrapper>(); + col = new ArrayList<>(); clusters.put( clust, col ); } col.add( wr ); } } - List<ModuleWrapper> wrapperList = new ArrayList<ModuleWrapper>( moduleDefinitions.keySet() ); + List<ModuleWrapper> wrapperList = new ArrayList<>( moduleDefinitions.keySet() ); int count = wrapperList.size() + 1; int index = 0; File javadocRoot = null; @@ -477,7 +448,7 @@ public class PopulateRepositoryMojo } } - List<ExternalsWrapper> externals = new ArrayList<ExternalsWrapper>(); + List<ExternalsWrapper> externals = new ArrayList<>(); try { for ( Map.Entry<ModuleWrapper, Artifact> elem : moduleDefinitions.entrySet() ) @@ -486,7 +457,7 @@ public class PopulateRepositoryMojo Artifact art = elem.getValue(); index = index + 1; getLog().info( "Processing " + index + "/" + count ); - File pom = createMavenProject( man, wrapperList, externals, searcher ); + File pom = createMavenProject( man, wrapperList, externals ); ArtifactMetadata metadata = new ProjectArtifactMetadata( art, pom ); art.addMetadata( metadata ); File javadoc = null; @@ -784,7 +755,7 @@ public class PopulateRepositoryMojo } private File createMavenProject( ModuleWrapper wrapper, List<ModuleWrapper> wrapperList, - List<ExternalsWrapper> externalsList, IndexSearcher searcher ) + List<ExternalsWrapper> externalsList ) throws MojoExecutionException { Model mavenModel = new Model(); @@ -799,7 +770,7 @@ public class PopulateRepositoryMojo mavenModel.setParent( artefactParent ); } ExamineManifest man = wrapper.getModuleManifest(); - List<Dependency> deps = new ArrayList<Dependency>(); + List<Dependency> deps = new ArrayList<>(); if ( !man.getDependencyTokens().isEmpty() ) { for ( String elem : man.getDependencyTokens() ) @@ -899,7 +870,7 @@ public class PopulateRepositoryMojo File f = new File( wrapper.getFile().getParentFile(), path ); if ( f.exists() ) { - Dependency dep = findExternal( searcher, f ); + Dependency dep = findExternal( f ); if ( dep != null ) { deps.add( dep ); @@ -963,61 +934,38 @@ public class PopulateRepositoryMojo return fil; } - private Dependency findExternal( IndexSearcher searcher, File f ) + private Dependency findExternal( File f ) { - if ( searcher == null ) + if ( externallist == null ) { return null; } try { + List<String> content512 = Files.readAllLines( externallist.toPath() ); MessageDigest shaDig = MessageDigest.getInstance( "SHA1" ); - InputStream is = new FileInputStream( f ); - try + + try ( InputStream is = new FileInputStream( f ); OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig ); ) { - OutputStream os = new DigestOutputStream( new NullOutputStream(), shaDig ); IOUtil.copy( is, os ); - os.close(); } - finally + String sha1 = encode ( shaDig.digest() ).toUpperCase(); + for ( String string : content512 ) { - is.close(); - } - String sha = encode( shaDig.digest() ); - TermQuery q = new TermQuery( new Term( "1", sha ) ); - TopScoreDocCollector collector = TopScoreDocCollector.create( 5 ); - searcher.search( q, collector ); - ScoreDoc[] hits = collector.topDocs().scoreDocs; - if ( hits.length >= 1 ) - { - int docId = hits[0].doc; - Document doc = searcher.doc( docId ); - IndexableField idField = doc.getField( "u" ); - if ( idField != null ) + String[] split = string.split( ";" ); + if ( split[0].equals( sha1 ) && split[1].contains( ":" ) ) { - String id = idField.stringValue(); - String[] splits = StringUtils.split( id, "|" ); + String[] splits = split[1].split( ":" ); Dependency dep = new Dependency(); dep.setArtifactId( splits[1] ); dep.setGroupId( splits[0] ); dep.setVersion( splits[2] ); dep.setType( "jar" ); - if ( splits.length > 3 && !"NA".equals( splits[3] ) ) - { - dep.setClassifier( splits[3] ); - } getLog().info( "found match " + splits[0] + ":" + splits[1] + ":" + splits[2] + " for " + f.getName() ); return dep; - } - else - { - getLog().error( "no idField for " + q ); - } - } - else - { - getLog().info( "no repository match for " + f.getName() ); + } } + getLog().info( "no repository match for " + f.getName() + f.getAbsolutePath() + " with sha " + sha1 ); } catch ( Exception x ) { @@ -1089,7 +1037,7 @@ public class PopulateRepositoryMojo { mavenModel.setParent( artefactParent ); } - List<Dependency> deps = new ArrayList<Dependency>(); + List<Dependency> deps = new ArrayList<>(); for ( ModuleWrapper wr : mods ) { Dependency dep = new Dependency(); @@ -1227,12 +1175,12 @@ public class PopulateRepositoryMojo List<Dependency> deps; - public ModuleWrapper( String module ) + ModuleWrapper( String module ) { this.module = module; } - public ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil ) + ModuleWrapper( String art, String ver, String grp, ExamineManifest manifest, File fil ) { man = manifest; artifact = art; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@netbeans.apache.org For additional commands, e-mail: commits-h...@netbeans.apache.org For further information about the NetBeans mailing lists, visit: https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists