MINDEXER-100: forward port 5.x changes to master Squashed from https://github.com/apache/maven-indexer/pull/14
Project: http://git-wip-us.apache.org/repos/asf/maven-indexer/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-indexer/commit/6cb2fcbe Tree: http://git-wip-us.apache.org/repos/asf/maven-indexer/tree/6cb2fcbe Diff: http://git-wip-us.apache.org/repos/asf/maven-indexer/diff/6cb2fcbe Branch: refs/heads/master Commit: 6cb2fcbe47883cd0decac0295493bef6577d7eb4 Parents: a7aeb9f Author: Tamas Cservenak <ta...@cservenak.net> Authored: Fri Mar 31 14:31:35 2017 +0200 Committer: Tamas Cservenak <ta...@cservenak.net> Committed: Fri Mar 31 14:31:35 2017 +0200 ---------------------------------------------------------------------- .../maven/index/context/NexusIndexWriter.java | 29 +- .../maven/index/updater/IndexDataReader.java | 15 +- .../apache/maven/index/updater/WagonHelper.java | 4 + .../org.apache.karaf.features.command-2.2.2.pom | 120 +++++- indexer-reader/README.md | 8 + indexer-reader/header.txt | 17 + indexer-reader/pom.xml | 66 ++++ .../apache/maven/index/reader/ChunkReader.java | 259 +++++++++++++ .../apache/maven/index/reader/ChunkWriter.java | 177 +++++++++ .../apache/maven/index/reader/IndexReader.java | 272 +++++++++++++ .../apache/maven/index/reader/IndexWriter.java | 196 ++++++++++ .../org/apache/maven/index/reader/Record.java | 384 +++++++++++++++++++ .../maven/index/reader/RecordCompactor.java | 207 ++++++++++ .../maven/index/reader/RecordExpander.java | 229 +++++++++++ .../maven/index/reader/ResourceHandler.java | 51 +++ .../org/apache/maven/index/reader/Utils.java | 170 ++++++++ .../index/reader/WritableResourceHandler.java | 58 +++ .../org/apache/maven/index/reader/packageinfo | 1 + .../index/reader/CachingResourceHandler.java | 129 +++++++ .../maven/index/reader/ChunkReaderTest.java | 95 +++++ .../index/reader/DirectoryResourceHandler.java | 87 +++++ .../maven/index/reader/HttpResourceHandler.java | 75 ++++ .../maven/index/reader/IndexReaderTest.java | 209 ++++++++++ .../maven/index/reader/IndexWriterTest.java | 93 +++++ .../apache/maven/index/reader/TestSupport.java | 184 +++++++++ .../apache/maven/index/reader/TestUtils.java | 108 ++++++ .../maven/index/reader/TransformTest.java | 98 +++++ .../simple/nexus-maven-repository-index.gz | Bin 0 -> 319 bytes .../nexus-maven-repository-index.properties | 6 + pom.xml | 33 +- 30 files changed, 3351 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java ---------------------------------------------------------------------- diff --git a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java index 038eaa8..130ee88 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java +++ b/indexer-core/src/main/java/org/apache/maven/index/context/NexusIndexWriter.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; @@ -38,16 +39,27 @@ import org.apache.lucene.util.Version; public class NexusIndexWriter extends IndexWriter { + public interface IndexWriterConfigFactory { + IndexWriterConfig create(Analyzer analyzer); + } + + public static IndexWriterConfigFactory CONFIG_FACTORY = new IndexWriterConfigFactory() { + public IndexWriterConfig create(final Analyzer analyzer) { + IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, analyzer ); + config.setRAMBufferSizeMB( 2.0 ); // old default + config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially + config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT); + return config; + } + }; + @Deprecated public NexusIndexWriter( final Directory directory, final Analyzer analyzer, boolean create ) throws CorruptIndexException, LockObtainFailedException, IOException { - //super( directory, analyzer, create, MaxFieldLength.LIMITED ); - this(directory, new IndexWriterConfig(Version.LUCENE_46, analyzer)); - - // setSimilarity( new NexusSimilarity() ); + this(directory, CONFIG_FACTORY.create(analyzer).setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND)); } - + public NexusIndexWriter( final Directory directory, final IndexWriterConfig config ) throws CorruptIndexException, LockObtainFailedException, IOException { @@ -58,11 +70,6 @@ public class NexusIndexWriter public static IndexWriterConfig defaultConfig() { - final IndexWriterConfig config = new IndexWriterConfig( Version.LUCENE_46, new NexusAnalyzer() ); - // default open mode is CreateOrAppend which suits us - config.setRAMBufferSizeMB( 2.0 ); // old default - config.setMergeScheduler( new SerialMergeScheduler() ); // merging serially - config.setWriteLockTimeout(IndexWriterConfig.WRITE_LOCK_TIMEOUT); - return config; + return CONFIG_FACTORY.create(new NexusAnalyzer()); } } http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java ---------------------------------------------------------------------- diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java index f76200c..f55fbf9 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java +++ b/indexer-core/src/main/java/org/apache/maven/index/updater/IndexDataReader.java @@ -9,7 +9,7 @@ package org.apache.maven.index.updater; * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -41,18 +41,17 @@ import org.apache.maven.index.context.IndexingContext; /** * An index data reader used to parse transfer index format. - * + * * @author Eugene Kuleshov */ public class IndexDataReader { private final DataInputStream dis; - public IndexDataReader( InputStream is ) + public IndexDataReader( final InputStream is ) throws IOException { BufferedInputStream bis = new BufferedInputStream( is, 1024 * 8 ); - // MINDEXER-13 // LightweightHttpWagon may have performed automatic decompression // Handle it transparently @@ -61,7 +60,7 @@ public class IndexDataReader if ( bis.read() == 0x1f && bis.read() == 0x8b ) // GZIPInputStream.GZIP_MAGIC { bis.reset(); - data = new GZIPInputStream( bis, 2 * 1024 ); + data = new GZIPInputStream( bis, 1024 * 8 ); } else { @@ -97,7 +96,7 @@ public class IndexDataReader } w.commit(); - + w.forceMerge(1); w.commit(); @@ -317,7 +316,7 @@ public class IndexDataReader /** * Reads index content by using a visitor. <br> * The visitor is called for each read documents after it has been populated with Lucene fields. - * + * * @param visitor an index data visitor * @param context indexing context * @return statistics about read data @@ -361,7 +360,7 @@ public class IndexDataReader /** * Called on each read document. The document is already populated with fields. - * + * * @param document read document */ void visitDocument( Document document ); http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java ---------------------------------------------------------------------- diff --git a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java index 0e450c9..40ea1d9 100644 --- a/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java +++ b/indexer-core/src/main/java/org/apache/maven/index/updater/WagonHelper.java @@ -198,6 +198,7 @@ public class WagonHelper throws IOException, FileNotFoundException { final File target = File.createTempFile( name, "" ); + target.deleteOnExit(); retrieve( name, target ); return new FileInputStream( target ) { @@ -220,6 +221,7 @@ public class WagonHelper } catch ( AuthorizationException e ) { + targetFile.delete(); String msg = "Authorization exception retrieving " + name; logError( msg, e ); IOException ioException = new IOException( msg ); @@ -228,6 +230,7 @@ public class WagonHelper } catch ( ResourceDoesNotExistException e ) { + targetFile.delete(); String msg = "Resource " + name + " does not exist"; logError( msg, e ); FileNotFoundException fileNotFoundException = new FileNotFoundException( msg ); @@ -236,6 +239,7 @@ public class WagonHelper } catch ( WagonException e ) { + targetFile.delete(); String msg = "Transfer for " + name + " failed"; logError( msg, e ); IOException ioException = new IOException( msg + "; " + e.getMessage() ); http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom ---------------------------------------------------------------------- diff --git a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom index 67e206f..062383c 100644 --- a/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom +++ b/indexer-core/src/test/repo-with-osgi/org/apache/karaf/features/org.apache.karaf.features.command/2.2.2/org.apache.karaf.features.command-2.2.2.pom @@ -1,5 +1,115 @@ -#Generated by org.apache.felix.bundleplugin -#Thu Jun 30 15:25:37 NDT 2011 -version=2.2.2 -groupId=org.apache.karaf.features -artifactId=org.apache.karaf.features.command +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <!-- + + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + --> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.karaf.features</groupId> + <artifactId>features</artifactId> + <version>2.2.2</version> + </parent> + <artifactId>org.apache.karaf.features.command</artifactId> + <packaging>bundle</packaging> + <name>Apache Karaf :: Features :: Command</name> + <description> + This bundle provides the Karaf shell commands to manipulate features. + </description> + <properties> + <appendedResourcesDirectory>${basedir}/../../etc/appended-resources</appendedResourcesDirectory> + </properties> + <dependencies> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.core</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.compendium</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.karaf.features</groupId> + <artifactId>org.apache.karaf.features.core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.bundlerepository</artifactId> + </dependency> + <dependency> + <groupId>org.apache.karaf.shell</groupId> + <artifactId>org.apache.karaf.shell.console</artifactId> + </dependency> + <dependency> + <groupId>org.apache.karaf.shell</groupId> + <artifactId>org.apache.karaf.shell.obr</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.osgi</groupId> + <artifactId>spring-osgi-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.servicemix.bundles</groupId> + <artifactId>org.apache.servicemix.bundles.junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <resources> + <resource> + <directory>${project.basedir}/src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + <resource> + <directory>${project.basedir}/src/main/resources</directory> + <filtering>true</filtering> + <includes> + <include>**/*.info</include> + </includes> + </resource> + </resources> + <plugins> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <configuration> + <instructions> + <Import-Package> + !${project.artifactId}*, javax.management, javax.management.loading, org.apache.felix.service.command, org.apache.felix.gogo.commands, org.apache.karaf.shell.console, * + </Import-Package> + <Private-Package>!*</Private-Package> + </instructions> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/README.md ---------------------------------------------------------------------- diff --git a/indexer-reader/README.md b/indexer-reader/README.md new file mode 100644 index 0000000..95aa1b8 --- /dev/null +++ b/indexer-reader/README.md @@ -0,0 +1,8 @@ +Indexer Reader Notes +================== + +Indexer Reader is a minimal dep-less library that is able to read published (remote) +index with incremental update support, making this library user able to integrate +published Maven Indexes into any engine without depending on maven-indexer-core +and it's transitive dependencies. + http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/header.txt ---------------------------------------------------------------------- diff --git a/indexer-reader/header.txt b/indexer-reader/header.txt new file mode 100644 index 0000000..1a2ef73 --- /dev/null +++ b/indexer-reader/header.txt @@ -0,0 +1,17 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/pom.xml ---------------------------------------------------------------------- diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml new file mode 100644 index 0000000..3f91847 --- /dev/null +++ b/indexer-reader/pom.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.indexer</groupId> + <artifactId>maven-indexer</artifactId> + <version>6.0-SNAPSHOT</version> + </parent> + + <artifactId>indexer-reader</artifactId> + <packaging>bundle</packaging> + + <name>Maven :: Indexer Reader</name> + <description> + Indexer Reader is a minimal dep-less library that is able to read published (remote) index with incremental update + support, making user able to integrate published Maven Indexes into any engine without depending on + maven-indexer-core and it's transitive dependencies. + </description> + + <dependencies> + <!-- Test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>18.0</version> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <version>3.0.0</version> + <extensions>true</extensions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java new file mode 100644 index 0000000..a6c75ac --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java @@ -0,0 +1,259 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UTFDataFormatException; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.zip.GZIPInputStream; + +/** + * Maven 2 Index published binary chunk reader, it reads raw Maven Indexer records from the transport binary format. + * + * @since 5.1.2 + */ +public class ChunkReader + implements Closeable, Iterable<Map<String, String>> +{ + private final String chunkName; + + private final DataInputStream dataInputStream; + + private final int version; + + private final Date timestamp; + + public ChunkReader(final String chunkName, final InputStream inputStream) throws IOException + { + this.chunkName = chunkName.trim(); + this.dataInputStream = new DataInputStream(new GZIPInputStream(inputStream, 2 * 1024)); + this.version = ((int) dataInputStream.readByte()) & 0xff; + this.timestamp = new Date(dataInputStream.readLong()); + } + + /** + * Returns the chunk name. + */ + public String getName() { + return chunkName; + } + + /** + * Returns index version. All releases so far always returned {@code 1}. + */ + public int getVersion() { + return version; + } + + /** + * Returns the index timestamp of last update of the index. + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * Returns the {@link Record} iterator. + */ + public Iterator<Map<String, String>> iterator() { + try { + return new IndexIterator(dataInputStream); + } + catch (IOException e) { + throw new RuntimeException("error", e); + } + } + + /** + * Closes this reader and it's underlying input. + */ + public void close() throws IOException { + dataInputStream.close(); + } + + /** + * Low memory footprint index iterator that incrementally parses the underlying stream. + */ + private static class IndexIterator + implements Iterator<Map<String, String>> + { + private final DataInputStream dataInputStream; + + private Map<String, String> nextRecord; + + public IndexIterator(final DataInputStream dataInputStream) throws IOException { + this.dataInputStream = dataInputStream; + this.nextRecord = nextRecord(); + } + + public boolean hasNext() { + return nextRecord != null; + } + + public Map<String, String> next() { + if (nextRecord == null) { + throw new NoSuchElementException("chunk depleted"); + } + Map<String, String> result = nextRecord; + nextRecord = nextRecord(); + return result; + } + + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + private Map<String, String> nextRecord() { + try { + return readRecord(dataInputStream); + } + catch (IOException e) { + throw new RuntimeException("read error", e); + } + } + } + + /** + * Reads and returns next record from the underlying stream, or {@code null} if no more records. + */ + private static Map<String, String> readRecord(final DataInput dataInput) + throws IOException + { + int fieldCount; + try { + fieldCount = dataInput.readInt(); + } + catch (EOFException ex) { + return null; // no more documents + } + + Map<String, String> recordMap = new HashMap<String, String>(); + for (int i = 0; i < fieldCount; i++) { + readField(recordMap, dataInput); + } + return recordMap; + } + + private static void readField(final Map<String, String> record, final DataInput dataInput) + throws IOException + { + dataInput.readByte(); // flags: neglect them + String name = dataInput.readUTF(); + String value = readUTF(dataInput); + record.put(name, value); + } + + private static String readUTF(final DataInput dataInput) + throws IOException + { + int utflen = dataInput.readInt(); + + byte[] bytearr; + char[] chararr; + + try { + bytearr = new byte[utflen]; + chararr = new char[utflen]; + } + catch (OutOfMemoryError e) { + IOException ioex = new IOException("Index data content is corrupt"); + ioex.initCause(e); + throw ioex; + } + + int c, char2, char3; + int count = 0; + int chararr_count = 0; + + dataInput.readFully(bytearr, 0, utflen); + + while (count < utflen) { + c = bytearr[count] & 0xff; + if (c > 127) { + break; + } + count++; + chararr[chararr_count++] = (char) c; + } + + while (count < utflen) { + c = bytearr[count] & 0xff; + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + count++; + chararr[chararr_count++] = (char) c; + break; + + case 12: + case 13: + /* 110x xxxx 10xx xxxx */ + count += 2; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 1]; + if ((char2 & 0xC0) != 0x80) { + throw new UTFDataFormatException("malformed input around byte " + count); + } + chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); + break; + + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 2]; + char3 = bytearr[count - 1]; + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { + throw new UTFDataFormatException("malformed input around byte " + (count - 1)); + } + chararr[chararr_count++] = + (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); + break; + + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException("malformed input around byte " + count); + } + } + + // The number of chars produced may be less than utflen + return new String(chararr, 0, chararr_count); + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java new file mode 100644 index 0000000..24e4701 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkWriter.java @@ -0,0 +1,177 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.zip.GZIPOutputStream; + +/** + * Maven 2 Index published binary chunk writer, it writes raw Maven Indexer records to the transport binary format. + * + * @since 5.1.2 + */ +public class ChunkWriter + implements Closeable +{ + private static final int F_INDEXED = 1; + + private static final int F_TOKENIZED = 2; + + private static final int F_STORED = 4; + + private final String chunkName; + + private final DataOutputStream dataOutputStream; + + private final int version; + + private final Date timestamp; + + public ChunkWriter(final String chunkName, final OutputStream outputStream, final int version, final Date timestamp) + throws IOException + { + this.chunkName = chunkName.trim(); + this.dataOutputStream = new DataOutputStream(new GZIPOutputStream(outputStream, 2 * 1024)); + this.version = version; + this.timestamp = timestamp; + + dataOutputStream.writeByte(version); + dataOutputStream.writeLong(timestamp == null ? -1 : timestamp.getTime()); + } + + /** + * Returns the chunk name. + */ + public String getName() { + return chunkName; + } + + /** + * Returns index version. All releases so far always returned {@code 1}. + */ + public int getVersion() { + return version; + } + + /** + * Returns the index timestamp of last update of the index. + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * Writes out the record iterator and returns the written record count. + */ + public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException { + int written = 0; + while (iterator.hasNext()) { + writeRecord(iterator.next(), dataOutputStream); + written++; + } + return written; + } + + /** + * Closes this reader and it's underlying input. + */ + public void close() throws IOException { + dataOutputStream.close(); + } + + private static void writeRecord(final Map<String, String> record, final DataOutput dataOutput) + throws IOException + { + dataOutput.writeInt(record.size()); + for (Map.Entry<String, String> entry : record.entrySet()) { + writeField(entry.getKey(), entry.getValue(), dataOutput); + } + } + + private static void writeField(final String fieldName, final String fieldValue, final DataOutput dataOutput) + throws IOException + { + boolean isIndexed = !(fieldName.equals("i") || fieldName.equals("m")); + boolean isTokenized = !(fieldName.equals("i") + || fieldName.equals("m") + || fieldName.equals("1") + || fieldName.equals("px")); + int flags = (isIndexed ? F_INDEXED : 0) + (isTokenized ? F_TOKENIZED : 0) + F_STORED; + dataOutput.writeByte(flags); + dataOutput.writeUTF(fieldName); + writeUTF(fieldValue, dataOutput); + } + + private static void writeUTF(final String str, final DataOutput dataOutput) + throws IOException + { + int strlen = str.length(); + int utflen = 0; + int c; + // use charAt instead of copying String to char array + for (int i = 0; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + utflen++; + } + else if (c > 0x07FF) { + utflen += 3; + } + else { + utflen += 2; + } + } + dataOutput.writeInt(utflen); + byte[] bytearr = new byte[utflen]; + int count = 0; + int i = 0; + for (; i < strlen; i++) { + c = str.charAt(i); + if (!((c >= 0x0001) && (c <= 0x007F))) { + break; + } + bytearr[count++] = (byte) c; + } + for (; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0001) && (c <= 0x007F)) { + bytearr[count++] = (byte) c; + + } + else if (c > 0x07FF) { + bytearr[count++] = (byte) (0xE0 | ((c >> 12) & 0x0F)); + bytearr[count++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } + else { + bytearr[count++] = (byte) (0xC0 | ((c >> 6) & 0x1F)); + bytearr[count++] = (byte) (0x80 | ((c >> 0) & 0x3F)); + } + } + dataOutput.write(bytearr, 0, utflen); + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java new file mode 100644 index 0000000..85c5409 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -0,0 +1,272 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; + +import org.apache.maven.index.reader.ResourceHandler.Resource; + +import static org.apache.maven.index.reader.Utils.loadProperties; +import static org.apache.maven.index.reader.Utils.storeProperties; + +/** + * Maven 2 Index reader that handles incremental updates if possible and provides one or more {@link ChunkReader}s, to + * read all the required records. + * + * @since 5.1.2 + */ +public class IndexReader + implements Iterable<ChunkReader>, Closeable +{ + private final WritableResourceHandler local; + + private final ResourceHandler remote; + + private final Properties localIndexProperties; + + private final Properties remoteIndexProperties; + + private final String indexId; + + private final Date publishedTimestamp; + + private final boolean incremental; + + private final List<String> chunkNames; + + public IndexReader(final WritableResourceHandler local, final ResourceHandler remote) throws IOException { + if (remote == null) { + throw new NullPointerException("remote resource handler null"); + } + this.local = local; + this.remote = remote; + remoteIndexProperties = loadProperties(remote.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (remoteIndexProperties == null) { + throw new IllegalArgumentException("Non-existent remote index"); + } + try { + if (local != null) { + Properties localProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (localProperties != null) { + this.localIndexProperties = localProperties; + String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { + throw new IllegalArgumentException( + "local and remote index IDs does not match or is null: " + localIndexId + ", " + + remoteIndexId); + } + this.indexId = localIndexId; + this.incremental = canRetrieveAllChunks(); + } + else { + localIndexProperties = null; + this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); + this.incremental = false; + } + } + else { + localIndexProperties = null; + this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); + this.incremental = false; + } + this.publishedTimestamp = Utils.INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); + this.chunkNames = calculateChunkNames(); + } + catch (ParseException e) { + IOException ex = new IOException("Index properties corrupted"); + ex.initCause(e); + throw ex; + } + } + + /** + * Returns the index context ID that published index has set. Usually it is equal to "repository ID" used in {@link + * Record.Type#DESCRIPTOR} but does not have to be. + */ + public String getIndexId() { + return indexId; + } + + /** + * Returns the {@link Date} when remote index was last published. + */ + public Date getPublishedTimestamp() { + return publishedTimestamp; + } + + /** + * Returns {@code true} if incremental update is about to happen. If incremental update, the {@link #iterator()} will + * return only the diff from the last update. + */ + public boolean isIncremental() { + return incremental; + } + + /** + * Returns unmodifiable list of actual chunks that needs to be pulled from remote {@link ResourceHandler}. Those are + * incremental chunks or the big main file, depending on result of {@link #isIncremental()}. Empty list means local + * index is up to date, and {@link #iterator()} will return empty iterator. + */ + public List<String> getChunkNames() { + return chunkNames; + } + + /** + * Closes the underlying {@link ResourceHandler}s. In case of incremental update use, it also assumes that user + * consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents to + * prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this method + * should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of + * this class should be closed manually. + */ + public void close() throws IOException { + remote.close(); + if (local != null) { + try { + syncLocalWithRemote(); + } + finally { + local.close(); + } + } + } + + /** + * Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental) + * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current + * {@link ChunkReader} if aborting. + */ + public Iterator<ChunkReader> iterator() { + return new ChunkReaderIterator(remote, chunkNames.iterator()); + } + + /** + * Stores the remote index properties into local index properties, preparing local {@link WritableResourceHandler} + * for future incremental updates. + */ + private void syncLocalWithRemote() throws IOException { + storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), remoteIndexProperties); + } + + /** + * Calculates the chunk names that needs to be fetched. + */ + private List<String> calculateChunkNames() { + if (incremental) { + ArrayList<String> chunkNames = new ArrayList<String>(); + int maxCounter = Integer.parseInt(remoteIndexProperties.getProperty("nexus.index.last-incremental")); + int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + currentCounter++; + while (currentCounter <= maxCounter) { + chunkNames.add(Utils.INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz"); + } + return Collections.unmodifiableList(chunkNames); + } + else { + return Collections.singletonList(Utils.INDEX_FILE_PREFIX + ".gz"); + } + } + + /** + * Verifies incremental update is possible, as all the diff chunks we need are still enlisted in remote properties. + */ + private boolean canRetrieveAllChunks() + { + String localChainId = localIndexProperties.getProperty("nexus.index.chain-id"); + String remoteChainId = remoteIndexProperties.getProperty("nexus.index.chain-id"); + + // If no chain id, or not the same, do full update + if (localChainId == null || remoteChainId == null || !localChainId.equals(remoteChainId)) { + return false; + } + + try { + int localLastIncremental = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + String currentLocalCounter = String.valueOf(localLastIncremental); + String nextLocalCounter = String.valueOf(localLastIncremental + 1); + // check remote props for existence of current or next chunk after local + for (Object key : remoteIndexProperties.keySet()) { + String sKey = (String) key; + if (sKey.startsWith("nexus.index.incremental-")) { + String value = remoteIndexProperties.getProperty(sKey); + if (currentLocalCounter.equals(value) || nextLocalCounter.equals(value)) { + return true; + } + } + } + } + catch (NumberFormatException e) { + // fall through + } + return false; + } + + /** + * Internal iterator implementation that lazily opens and closes the returned {@link ChunkReader}s as this iterator + * is being consumed. + */ + private static class ChunkReaderIterator + implements Iterator<ChunkReader> + { + private final ResourceHandler resourceHandler; + + private final Iterator<String> chunkNamesIterator; + + private Resource currentResource; + + private ChunkReader currentChunkReader; + + private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator) { + this.resourceHandler = resourceHandler; + this.chunkNamesIterator = chunkNamesIterator; + } + + public boolean hasNext() { + return chunkNamesIterator.hasNext(); + } + + public ChunkReader next() { + String chunkName = chunkNamesIterator.next(); + try { + if (currentChunkReader != null) { + currentChunkReader.close(); + } + currentResource = resourceHandler.locate(chunkName); + currentChunkReader = new ChunkReader(chunkName, currentResource.read()); + return currentChunkReader; + } + catch (IOException e) { + throw new RuntimeException("IO problem while switching chunk readers", e); + } + } + + public void remove() { + throw new UnsupportedOperationException("remove"); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java new file mode 100644 index 0000000..68d4abb --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexWriter.java @@ -0,0 +1,196 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.text.ParseException; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import org.apache.maven.index.reader.WritableResourceHandler.WritableResource; + +import static org.apache.maven.index.reader.Utils.loadProperties; +import static org.apache.maven.index.reader.Utils.storeProperties; + +/** + * Maven 2 Index writer that writes chunk and maintains published property file. + * <p/> + * <strong>Currently no incremental update is supported, as the deleteion states should be maintained by + * caller</strong>. Hence, this writer will always produce the "main" chunk only. + * + * @since 5.1.2 + */ +public class IndexWriter + implements Closeable +{ + private static final int INDEX_V1 = 1; + + private final WritableResourceHandler local; + + private final Properties localIndexProperties; + + private final boolean incremental; + + private final String nextChunkCounter; + + private final String nextChunkName; + + public IndexWriter(final WritableResourceHandler local, final String indexId, final boolean incrementalSupported) + throws IOException + { + if (local == null) { + throw new NullPointerException("local resource handler null"); + } + if (indexId == null) { + throw new NullPointerException("indexId null"); + } + this.local = local; + Properties indexProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties")); + if (incrementalSupported && indexProperties != null) { + this.localIndexProperties = indexProperties; + // existing index, this is incremental publish, and we will add new chunk + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (localIndexId == null || !localIndexId.equals(indexId)) { + throw new IllegalArgumentException( + "index already exists and indexId mismatch or unreadable: " + localIndexId + ", " + + indexId); + } + this.incremental = true; + this.nextChunkCounter = calculateNextChunkCounter(); + this.nextChunkName = Utils.INDEX_FILE_PREFIX + "." + nextChunkCounter + ".gz"; + } + else { + // non-existing index, create published index from scratch + this.localIndexProperties = new Properties(); + this.localIndexProperties.setProperty("nexus.index.id", indexId); + this.localIndexProperties.setProperty("nexus.index.chain-id", UUID.randomUUID().toString()); + this.incremental = false; + this.nextChunkCounter = null; + this.nextChunkName = Utils.INDEX_FILE_PREFIX + ".gz"; + } + } + + /** + * Returns the index context ID that published index has set. + */ + public String getIndexId() { + return localIndexProperties.getProperty("nexus.index.id"); + } + + /** + * Returns the {@link Date} when index was last published or {@code null} if this is first publishing. In other + * words,returns {@code null} when {@link #isIncremental()} returns {@code false}. After this writer is closed, the + * return value is updated to "now" (in {@link #close() method}. + */ + public Date getPublishedTimestamp() { + try { + String timestamp = localIndexProperties.getProperty("nexus.index.timestamp"); + if (timestamp != null) { + return Utils.INDEX_DATE_FORMAT.parse(timestamp); + } + return null; + } + catch (ParseException e) { + throw new RuntimeException("Corrupt date", e); + } + } + + /** + * Returns {@code true} if incremental publish is about to happen. + */ + public boolean isIncremental() { + return incremental; + } + + /** + * Returns the chain id of published index. If {@link #isIncremental()} is {@code false}, this is the newly generated + * chain ID. + */ + public String getChainId() { + return localIndexProperties.getProperty("nexus.index.chain-id"); + } + + /** + * Returns the next chunk name about to be published. + */ + public String getNextChunkName() { + return nextChunkName; + } + + /** + * Writes out the record iterator and returns the written record count. + */ + public int writeChunk(final Iterator<Map<String, String>> iterator) throws IOException { + int written; + WritableResource writableResource = local.locate(nextChunkName); + try { + final ChunkWriter chunkWriter = new ChunkWriter(nextChunkName, writableResource.write(), INDEX_V1, new Date()); + try { + written = chunkWriter.writeChunk(iterator); + } + finally { + chunkWriter.close(); + } + if (incremental) { + // TODO: update main gz file + } + return written; + } + finally { + writableResource.close(); + } + } + + /** + * Closes the underlying {@link ResourceHandler} and synchronizes published index properties, so remote clients + * becomes able to consume newly published index. If sync is not desired (ie. due to aborted publish), then this + * method should NOT be invoked, but rather the {@link ResourceHandler} that caller provided in constructor of + * this class should be closed manually. + */ + public void close() throws IOException { + try { + if (incremental) { + localIndexProperties.setProperty("nexus.index.last-incremental", nextChunkCounter); + } + localIndexProperties.setProperty("nexus.index.timestamp", Utils.INDEX_DATE_FORMAT.format(new Date())); + storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), localIndexProperties); + } + finally { + local.close(); + } + } + + /** + * Calculates the chunk names that needs to be fetched. + */ + private String calculateNextChunkCounter() { + String lastChunkCounter = localIndexProperties.getProperty("nexus.index.last-incremental"); + if (lastChunkCounter != null) { + return String.valueOf(Integer.parseInt(lastChunkCounter) + 1); + } + else { + return "1"; + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java new file mode 100644 index 0000000..1746845 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -0,0 +1,384 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Map; + +/** + * Maven 2 Index record. + * + * @since 5.1.2 + */ +public final class Record +{ + public static final class EntryKey<T> + { + private final String name; + + private final Class<T> proto; + + public EntryKey(final String name, final Class<T> proto) { + if (name == null) { + throw new NullPointerException("name is null"); + } + if (proto == null) { + throw new NullPointerException("proto is null"); + } + this.name = name; + this.proto = proto; + } + + public T coerce(final Object object) { + return (T) proto.cast(object); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EntryKey)) { + return false; + } + EntryKey entryKey = (EntryKey) o; + return name.equals(entryKey.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return "Key{" + + "name='" + name + '\'' + + ", type=" + proto.getSimpleName() + + '}'; + } + } + + /** + * Key of repository ID entry, that contains {@link String}. + */ + public static final EntryKey<String> REPOSITORY_ID = new EntryKey<String>("repositoryId", String.class); + + /** + * Key of all groups list entry, that contains {@link java.util.List<String>}. + */ + public static final EntryKey<String[]> ALL_GROUPS = new EntryKey<String[]>("allGroups", String[].class); + + /** + * Key of root groups list entry, that contains {@link java.util.List<String>}. + */ + public static final EntryKey<String[]> ROOT_GROUPS = new EntryKey<String[]>("rootGroups", String[].class); + + /** + * Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link + * Long}. + */ + public static final EntryKey<Long> REC_MODIFIED = new EntryKey<Long>("recordModified", Long.class); + + /** + * Key of artifact groupId entry, that contains {@link String}. + */ + public static final EntryKey<String> GROUP_ID = new EntryKey<String>("groupId", String.class); + + /** + * Key of artifact artifactId entry, that contains {@link String}. + */ + public static final EntryKey<String> ARTIFACT_ID = new EntryKey<String>("artifactId", String.class); + + /** + * Key of artifact version entry, that contains {@link String}. + */ + public static final EntryKey<String> VERSION = new EntryKey<String>("version", String.class); + + /** + * Key of artifact classifier entry, that contains {@link String}. + */ + public static final EntryKey<String> CLASSIFIER = new EntryKey<String>("classifier", String.class); + + /** + * Key of artifact packaging entry, that contains {@link String}. + */ + public static final EntryKey<String> PACKAGING = new EntryKey<String>("packaging", String.class); + + /** + * Key of artifact file extension, that contains {@link String}. + */ + public static final EntryKey<String> FILE_EXTENSION = new EntryKey<String>("fileExtension", String.class); + + /** + * Key of artifact file last modified timestamp, that contains {@link Long}. + */ + public static final EntryKey<Long> FILE_MODIFIED = new EntryKey<Long>("fileModified", Long.class); + + /** + * Key of artifact file size in bytes, that contains {@link Long}. + */ + public static final EntryKey<Long> FILE_SIZE = new EntryKey<Long>("fileSize", Long.class); + + /** + * Key of artifact Sources presence flag, that contains {@link Boolean}. + */ + public static final EntryKey<Boolean> HAS_SOURCES = new EntryKey<Boolean>("hasSources", Boolean.class); + + /** + * Key of artifact Javadoc presence flag, that contains {@link Boolean}. + */ + public static final EntryKey<Boolean> HAS_JAVADOC = new EntryKey<Boolean>("hasJavadoc", Boolean.class); + + /** + * Key of artifact signature presence flag, that contains {@link Boolean}. + */ + public static final EntryKey<Boolean> HAS_SIGNATURE = new EntryKey<Boolean>("hasSignature", Boolean.class); + + /** + * Key of artifact name (as set in POM), that contains {@link String}. + */ + public static final EntryKey<String> NAME = new EntryKey<String>("name", String.class); + + /** + * Key of artifact description (as set in POM), that contains {@link String}. + */ + public static final EntryKey<String> DESCRIPTION = new EntryKey<String>("description", String.class); + + /** + * Key of artifact SHA1 digest, that contains {@link String}. + */ + public static final EntryKey<String> SHA1 = new EntryKey<String>("sha1", String.class); + + /** + * Key of artifact contained class names, that contains {@link java.util.List<String>}. Extracted by {@code + * JarFileContentsIndexCreator}. + */ + public static final EntryKey<String[]> CLASSNAMES = new EntryKey<String[]>("classNames", String[].class); + + /** + * Key of plugin artifact prefix, that contains {@link String}. Extracted by {@code + * MavenPluginArtifactInfoIndexCreator}. + */ + public static final EntryKey<String> PLUGIN_PREFIX = new EntryKey<String>("pluginPrefix", String.class); + + /** + * Key of plugin artifact goals, that contains {@link java.util.List<String>}. Extracted by {@code + * MavenPluginArtifactInfoIndexCreator}. + */ + public static final EntryKey<String[]> PLUGIN_GOALS = new EntryKey<String[]>("pluginGoals", String[].class); + + /** + * Key of OSGi "Bundle-SymbolicName" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_BUNDLE_SYMBOLIC_NAME = new EntryKey<String>("Bundle-SymbolicName", + String.class); + + /** + * Key of OSGi "Bundle-Version" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_BUNDLE_VERSION = new EntryKey<String>("Bundle-Version", String.class); + + /** + * Key of OSGi "Export-Package" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_EXPORT_PACKAGE = new EntryKey<String>("Export-Package", String.class); + + /** + * Key of OSGi "Export-Service" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_EXPORT_SERVICE = new EntryKey<String>("Export-Service", String.class); + + /** + * Key of OSGi "Bundle-Description" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_BUNDLE_DESCRIPTION = new EntryKey<String>("Bundle-Description", + String.class); + + /** + * Key of OSGi "Bundle-Name" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_BUNDLE_NAME = new EntryKey<String>("Bundle-Name", String.class); + + /** + * Key of OSGi "Bundle-License" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_BUNDLE_LICENSE = new EntryKey<String>("Bundle-License", String.class); + + /** + * Key of OSGi "Bundle-DocURL" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_EXPORT_DOCURL = new EntryKey<String>("Bundle-DocURL", String.class); + + /** + * Key of OSGi "Import-Package" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_IMPORT_PACKAGE = new EntryKey<String>("Import-Package", String.class); + + /** + * Key of OSGi "Require-Bundle" manifest entry, that contains {@link String}. Extracted by {@code + * OsgiArtifactIndexCreator}. + */ + public static final EntryKey<String> OSGI_REQUIRE_BUNDLE = new EntryKey<String>("Require-Bundle", String.class); + + /** + * Types of returned records returned from index. + */ + public enum Type + { + /** + * Descriptor record. Can be safely ignored. + * Contains following entries: + * <ul> + * <li>{@link #REPOSITORY_ID}</li> + * </ul> + */ + DESCRIPTOR, + + /** + * Artifact ADD record. Records of this type should be added to your indexing system. + * Contains following entries: + * <ul> + * <li>{@link #REC_MODIFIED} (when record was added/modified on index)</li> + * <li>{@link #GROUP_ID}</li> + * <li>{@link #ARTIFACT_ID}</li> + * <li>{@link #VERSION}</li> + * <li>{@link #CLASSIFIER} (optional)</li> + * <li>{@link #FILE_EXTENSION}</li> + * <li>{@link #FILE_MODIFIED}</li> + * <li>{@link #FILE_SIZE}</li> + * <li>{@link #PACKAGING}</li> + * <li>{@link #HAS_SOURCES}</li> + * <li>{@link #HAS_JAVADOC}</li> + * <li>{@link #HAS_SIGNATURE}</li> + * <li>{@link #NAME}</li> + * <li>{@link #DESCRIPTION}</li> + * <li>{@link #SHA1}</li> + * <li>{@link #CLASSNAMES} (optional)</li> + * <li>{@link #PLUGIN_PREFIX} (optional, for maven-plugins only)</li> + * <li>{@link #PLUGIN_GOALS} (optional, for maven-plugins only)</li> + * </ul> + */ + ARTIFACT_ADD, + + /** + * Artifact REMOVE record. In case of incremental updates, signals that this artifact was removed. Records of this + * type should be removed from your indexing system. + * Contains following entries: + * <ul> + * <li>{@link #REC_MODIFIED} (when record was deleted from index)</li> + * <li>{@link #GROUP_ID}</li> + * <li>{@link #ARTIFACT_ID}</li> + * <li>{@link #VERSION}</li> + * <li>{@link #CLASSIFIER} (optional)</li> + * <li>{@link #FILE_EXTENSION} (if {@link #CLASSIFIER} present)</li> + * <li>{@link #PACKAGING} (optional)</li> + * </ul> + */ + ARTIFACT_REMOVE, + + /** + * Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored. + * Contains following entries: + * <ul> + * <li>{@link #ALL_GROUPS}</li> + * </ul> + */ + ALL_GROUPS, + + /** + * Special record, containing all the root groups of Maven "groupId"s that are enlisted on the index. Can be safely + * ignored. + * Contains following entries: + * <ul> + * <li>{@link #ROOT_GROUPS}</li> + * </ul> + */ + ROOT_GROUPS + } + + private final Type type; + + private final Map<EntryKey, Object> expanded; + + public Record(final Type type, final Map<EntryKey, Object> expanded) { + this.type = type; + this.expanded = expanded; + } + + /** + * Returns the {@link Type} of this record. Usually users would be interested in {@link Type#ARTIFACT_ADD} and {@link + * Type#ARTIFACT_REMOVE} types only to maintain their own index. Still, indexer offers extra records too, see {@link + * Type} for all existing types. + */ + public Type getType() { + return type; + } + + /** + * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption. + */ + public Map<EntryKey, Object> getExpanded() { + return expanded; + } + + /** + * Returns {@code true} if this record contains given {@link EntryKey}. + */ + boolean containsKey(final EntryKey<?> entryKey) { return expanded.containsKey(entryKey); } + + /** + * Type safe handy method to get value from expanded map. + */ + public <T> T get(final EntryKey<T> entryKey) { + return entryKey.coerce(expanded.get(entryKey)); + } + + /** + * Type safe handy method to put value to expanded map. Accepts {@code null} values, that removes the mapping. + */ + public <T> T put(final EntryKey<T> entryKey, final T value) { + if (value == null) { + return entryKey.coerce(expanded.remove(entryKey)); + } + else { + if (!entryKey.proto.isAssignableFrom(value.getClass())) { + throw new IllegalArgumentException("Key " + entryKey + " does not accepts value " + value); + } + return entryKey.coerce(expanded.put(entryKey, value)); + } + } + + @Override + public String toString() { + return "Record{" + + "type=" + type + + ", expanded=" + expanded + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java new file mode 100644 index 0000000..76fc1e5 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordCompactor.java @@ -0,0 +1,207 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Record.Type; + +import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; +import static org.apache.maven.index.reader.Utils.INFO; +import static org.apache.maven.index.reader.Utils.UINFO; +import static org.apache.maven.index.reader.Utils.nvl; + +/** + * Maven 2 Index record transformer, that transforms {@link Record}s into "native" Maven Indexer records. + * + * @since 5.1.2 + */ +public class RecordCompactor +{ + /** + * Compacts {@link Record} into low level MI record with all the encoded fields as physically present in MI binary + * chunk. + */ + public Map<String, String> apply(final Record record) { + if (Type.DESCRIPTOR == record.getType()) { + return compactDescriptor(record); + } + else if (Type.ALL_GROUPS == record.getType()) { + return compactAllGroups(record); + } + else if (Type.ROOT_GROUPS == record.getType()) { + return compactRootGroups(record); + } + else if (Type.ARTIFACT_REMOVE == record.getType()) { + return compactDeletedArtifact(record); + } + else if (Type.ARTIFACT_ADD == record.getType()) { + return compactAddedArtifact(record); + } + else { + throw new IllegalArgumentException("Unknown record: " + record); + } + } + + private static Map<String, String> compactDescriptor(final Record record) { + final Map<String, String> result = new HashMap<String, String>(); + result.put("DESCRIPTOR", "NexusIndex"); + result.put("IDXINFO", "1.0|" + record.get(Record.REPOSITORY_ID)); + return result; + } + + private static Map<String, String> compactAllGroups(final Record record) { + final Map<String, String> result = new HashMap<String, String>(); + result.put("allGroups", "allGroups"); + putIfNotNullAsStringArray(record.get(Record.ALL_GROUPS), result, "allGroupsList"); + return result; + } + + private static Map<String, String> compactRootGroups(final Record record) { + final Map<String, String> result = new HashMap<String, String>(); + result.put("rootGroups", "allGroups"); + putIfNotNullAsStringArray(record.get(Record.ROOT_GROUPS), result, "rootGroupsList"); + return result; + } + + private static Map<String, String> compactDeletedArtifact(final Record record) { + final Map<String, String> result = new HashMap<String, String>(); + putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m"); + result.put("del", compactUinfo(record)); + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private static Map<String, String> compactAddedArtifact(final Record record) { + final Map<String, String> result = new HashMap<String, String>(); + + // Minimal + result.put(UINFO, compactUinfo(record)); + + StringBuilder info = new StringBuilder(); + info.append(nvl(record.get(Record.PACKAGING))); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.FILE_MODIFIED)); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.FILE_SIZE)); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_SOURCES) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_JAVADOC) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(record.get(Record.HAS_SIGNATURE) ? "1" : "0"); + info.append(FIELD_SEPARATOR); + info.append(nvl(record.get(Record.FILE_EXTENSION))); + result.put(INFO, info.toString()); + + putIfNotNullTS(record.get(Record.REC_MODIFIED), result, "m"); + putIfNotNull(record.get(Record.NAME), result, "n"); + putIfNotNull(record.get(Record.DESCRIPTION), result, "d"); + putIfNotNull(record.get(Record.SHA1), result, "1"); + + // Jar file contents (optional) + putIfNotNullAsStringArray(record.get(Record.CLASSNAMES), result, "classnames"); + + // Maven Plugin (optional) + putIfNotNull(record.get(Record.PLUGIN_PREFIX), result, "px"); + putIfNotNullAsStringArray(record.get(Record.PLUGIN_GOALS), result, "gx"); + + // OSGi (optional) + putIfNotNull(record.get(Record.OSGI_BUNDLE_SYMBOLIC_NAME), result, "Bundle-SymbolicName"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_VERSION), result, "Bundle-Version"); + putIfNotNull(record.get(Record.OSGI_EXPORT_PACKAGE), result, "Export-Package"); + putIfNotNull(record.get(Record.OSGI_EXPORT_SERVICE), result, "Export-Service"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_DESCRIPTION), result, "Bundle-Description"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_NAME), result, "Bundle-Name"); + putIfNotNull(record.get(Record.OSGI_BUNDLE_LICENSE), result, "Bundle-License"); + putIfNotNull(record.get(Record.OSGI_EXPORT_DOCURL), result, "Bundle-DocURL"); + putIfNotNull(record.get(Record.OSGI_IMPORT_PACKAGE), result, "Import-Package"); + putIfNotNull(record.get(Record.OSGI_REQUIRE_BUNDLE), result, "Require-Bundle"); + + return result; + } + + /** + * Creates UINFO synthetic field. + */ + private static String compactUinfo(final Record record) { + final String classifier = record.get(Record.CLASSIFIER); + StringBuilder sb = new StringBuilder(); + sb.append(record.get(Record.GROUP_ID)) + .append(FIELD_SEPARATOR) + .append(record.get(Record.ARTIFACT_ID)) + .append(FIELD_SEPARATOR) + .append(record.get(Record.VERSION)) + .append(FIELD_SEPARATOR) + .append(nvl(classifier)); + if (classifier != null) { + sb.append(FIELD_SEPARATOR).append(record.get(Record.FILE_EXTENSION)); + } + return sb.toString(); + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final String source, + final Map<String, String> target, + final String targetName) + { + if (source != null) { + target.put(targetName, source); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Long source, + final Map<String, String> target, + final String targetName) + { + if (source != null) { + target.put(targetName, String.valueOf(source)); + } + } + + /** + * Helper to put a array value from source map into target map joined with {@link Utils#FIELD_SEPARATOR}, if not + * null. + */ + private static void putIfNotNullAsStringArray( + final String[] source, + final Map<String, String> target, + final String targetName) + { + if (source != null && source.length > 0) { + StringBuilder sb = new StringBuilder(); + sb.append(source[0]); + for (int i = 1; i < source.length; i++) { + sb.append(FIELD_SEPARATOR).append(source[i]); + } + target.put(targetName, sb.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java new file mode 100644 index 0000000..6d7f4be --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/RecordExpander.java @@ -0,0 +1,229 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Record.EntryKey; +import org.apache.maven.index.reader.Record.Type; + +import static org.apache.maven.index.reader.Utils.FIELD_SEPARATOR; +import static org.apache.maven.index.reader.Utils.FS_PATTERN; +import static org.apache.maven.index.reader.Utils.INFO; +import static org.apache.maven.index.reader.Utils.NOT_AVAILABLE; +import static org.apache.maven.index.reader.Utils.UINFO; +import static org.apache.maven.index.reader.Utils.renvl; + +/** + * Maven 2 Index record transformer, that transforms "native" Maven Indexer records into {@link Record}s. + * + * @since 5.1.2 + */ +public class RecordExpander +{ + /** + * Expands MI low level record into {@link Record}. + */ + public Record apply(final Map<String, String> recordMap) { + if (recordMap.containsKey("DESCRIPTOR")) { + return expandDescriptor(recordMap); + } + else if (recordMap.containsKey("allGroups")) { + return expandAllGroups(recordMap); + } + else if (recordMap.containsKey("rootGroups")) { + return expandRootGroups(recordMap); + } + else if (recordMap.containsKey("del")) { + return expandDeletedArtifact(recordMap); + } + else { + // Fix up UINFO field wrt MINDEXER-41 + final String uinfo = recordMap.get(UINFO); + final String info = recordMap.get(INFO); + if (uinfo != null && !(info == null || info.trim().length() == 0)) { + final String[] splitInfo = FS_PATTERN.split(info); + if (splitInfo.length > 6) { + final String extension = splitInfo[6]; + if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) { + recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension); + } + } + } + return expandAddedArtifact(recordMap); + } + } + + private static Record expandDescriptor(final Map<String, String> raw) { + final Record result = new Record(Type.DESCRIPTOR, new HashMap<EntryKey, Object>()); + String[] r = FS_PATTERN.split(raw.get("IDXINFO")); + result.put(Record.REPOSITORY_ID, r[1]); + return result; + } + + private static Record expandAllGroups(final Map<String, String> raw) { + final Record result = new Record(Type.ALL_GROUPS, new HashMap<EntryKey, Object>()); + putIfNotNullAsStringArray(raw, "allGroupsList", result, Record.ALL_GROUPS); + return result; + } + + private static Record expandRootGroups(final Map<String, String> raw) { + final Record result = new Record(Type.ROOT_GROUPS, new HashMap<EntryKey, Object>()); + putIfNotNullAsStringArray(raw, "rootGroupsList", result, Record.ROOT_GROUPS); + return result; + } + + private static Record expandDeletedArtifact(final Map<String, String> raw) { + final Record result = new Record(Type.ARTIFACT_REMOVE, new HashMap<EntryKey, Object>()); + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + if (raw.containsKey("del")) { + expandUinfo(raw.get("del"), result); + } + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private static Record expandAddedArtifact(final Map<String, String> raw) { + final Record result = new Record(Type.ARTIFACT_ADD, new HashMap<EntryKey, Object>()); + + // Minimal + expandUinfo(raw.get(UINFO), result); + final String info = raw.get(INFO); + if (info != null) { + String[] r = FS_PATTERN.split(info); + result.put(Record.PACKAGING, renvl(r[0])); + result.put(Record.FILE_MODIFIED, Long.valueOf(r[1])); + result.put(Record.FILE_SIZE, Long.valueOf(r[2])); + result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE : Boolean.FALSE); + result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE : Boolean.FALSE); + result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE : Boolean.FALSE); + if (r.length > 6) { + result.put(Record.FILE_EXTENSION, r[6]); + } + else { + final String packaging = Record.PACKAGING.coerce(result.get(Record.PACKAGING)); + if (result.containsKey(Record.CLASSIFIER) + || "pom".equals(packaging) + || "war".equals(packaging) + || "ear".equals(packaging)) { + result.put(Record.FILE_EXTENSION, packaging); + } + else { + result.put(Record.FILE_EXTENSION, "jar"); // best guess + } + } + } + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + putIfNotNull(raw, "n", result, Record.NAME); + putIfNotNull(raw, "d", result, Record.DESCRIPTION); + putIfNotNull(raw, "1", result, Record.SHA1); + + // Jar file contents (optional) + putIfNotNullAsStringArray(raw, "classnames", result, Record.CLASSNAMES); + + // Maven Plugin (optional) + putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX); + putIfNotNullAsStringArray(raw, "gx", result, Record.PLUGIN_GOALS); + + // OSGi (optional) + putIfNotNull(raw, "Bundle-SymbolicName", result, Record.OSGI_BUNDLE_SYMBOLIC_NAME); + putIfNotNull(raw, "Bundle-Version", result, Record.OSGI_BUNDLE_VERSION); + putIfNotNull(raw, "Export-Package", result, Record.OSGI_EXPORT_PACKAGE); + putIfNotNull(raw, "Export-Service", result, Record.OSGI_EXPORT_SERVICE); + putIfNotNull(raw, "Bundle-Description", result, Record.OSGI_BUNDLE_DESCRIPTION); + putIfNotNull(raw, "Bundle-Name", result, Record.OSGI_BUNDLE_NAME); + putIfNotNull(raw, "Bundle-License", result, Record.OSGI_BUNDLE_LICENSE); + putIfNotNull(raw, "Bundle-DocURL", result, Record.OSGI_EXPORT_DOCURL); + putIfNotNull(raw, "Import-Package", result, Record.OSGI_IMPORT_PACKAGE); + putIfNotNull(raw, "Require-Bundle", result, Record.OSGI_REQUIRE_BUNDLE); + + return result; + } + + /** + * Expands UINFO synthetic field. Handles {@code null} String inputs. + */ + private static void expandUinfo(final String uinfo, final Record result) { + if (uinfo != null) { + String[] r = FS_PATTERN.split(uinfo); + result.put(Record.GROUP_ID, r[0]); + result.put(Record.ARTIFACT_ID, r[1]); + result.put(Record.VERSION, r[2]); + String classifier = renvl(r[3]); + if (classifier != null) { + result.put(Record.CLASSIFIER, classifier); + if (r.length > 4) { + result.put(Record.FILE_EXTENSION, r[4]); + } + } + else if (r.length > 4) { + result.put(Record.PACKAGING, r[4]); + } + } + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final Map<String, String> source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, value); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Map<String, String> source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Long.valueOf(value)); + } + } + + /** + * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null. + */ + private static void putIfNotNullAsStringArray( + final Map<String, String> source, + final String sourceName, + final Record target, + final EntryKey targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, FS_PATTERN.split(value)); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/6cb2fcbe/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java new file mode 100644 index 0000000..244a994 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -0,0 +1,51 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; + +/** + * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that + * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). Is used by single + * thread only. + * + * @since 5.1.2 + */ +public interface ResourceHandler + extends Closeable +{ + interface Resource + { + /** + * Returns the {@link InputStream} stream of the resource, if exists, {@code null} otherwise. The stream should + * be closed by caller, otherwise resource leaks might be introduced. + */ + InputStream read() throws IOException; + } + + /** + * Returns the {@link Resource} with {@code name}, non {@code null}. + * + * @param name Resource name, guaranteed to be non-{@code null} and is FS and URL safe string. + */ + Resource locate(String name) throws IOException; +}
