This is an automated email from the ASF dual-hosted git repository. sjaranowski pushed a commit to branch MRESOLVER-351 in repository https://gitbox.apache.org/repos/asf/maven-resolver.git
commit a84d3eb10ad3c440cf1a88f9e9eb1cbe72c2e020 Author: Slawomir Jaranowski <[email protected]> AuthorDate: Wed Apr 5 23:50:56 2023 +0200 [MRESOLVER-351] Thread count as core multiplied Syntax as 1.5C for thread count parameters --- .../aether/util/concurrency/ExecutorUtils.java | 53 +++++++++++++++++++- .../aether/util/concurrency/ExecutorUtilsTest.java | 58 ++++++++++++++++++++++ src/site/markdown/configuration.md | 6 +-- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/maven-resolver-util/src/main/java/org/eclipse/aether/util/concurrency/ExecutorUtils.java b/maven-resolver-util/src/main/java/org/eclipse/aether/util/concurrency/ExecutorUtils.java index cb2a734b..20ac35f9 100644 --- a/maven-resolver-util/src/main/java/org/eclipse/aether/util/concurrency/ExecutorUtils.java +++ b/maven-resolver-util/src/main/java/org/eclipse/aether/util/concurrency/ExecutorUtils.java @@ -87,10 +87,61 @@ public final class ExecutorUtils { if (defaultValue < 1) { throw new IllegalArgumentException("Invalid defaultValue: " + defaultValue + ". Must be greater than 0."); } - int threadCount = ConfigUtils.getInteger(session, defaultValue, keys); + + int threadCount; + + Object threadCountAsObject = ConfigUtils.getObject(session, defaultValue, keys); + if (threadCountAsObject instanceof Number) { + threadCount = ((Number) threadCountAsObject).intValue(); + } else { + threadCount = calculateDegreeOfConcurrency(String.valueOf(threadCountAsObject)); + } + if (threadCount < 1) { throw new IllegalArgumentException("Invalid value: " + threadCount + ". Must be greater than 0."); } return threadCount; } + + /** + * Calculates "degree of concurrency" (count of threads to be used) based on non-null input string. String may + * be string representation of integer or a string representation of float followed by "C" character + * (case-sensitive) in which case the float is interpreted as multiplier for core count as reported by Java. + * Blatantly copied (and simplified) from maven-embedder + * {@code org.apache.maven.cli.MavenCli#calculateDegreeOfConcurrency} class. + */ + private static int calculateDegreeOfConcurrency(String threadConfiguration) { + if (threadConfiguration.endsWith("C")) { + threadConfiguration = threadConfiguration.substring(0, threadConfiguration.length() - 1); + + try { + float coreMultiplier = Float.parseFloat(threadConfiguration); + + if (coreMultiplier <= 0.0f) { + throw new IllegalArgumentException("Invalid threads core multiplier value: '" + threadConfiguration + + "C'. Value must be positive."); + } + + int threads = (int) (coreMultiplier * Runtime.getRuntime().availableProcessors()); + return threads == 0 ? 1 : threads; + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid threads value: '" + threadConfiguration + "C'. Value must be positive."); + } + } else { + try { + int threads = Integer.parseInt(threadConfiguration); + + if (threads <= 0) { + throw new IllegalArgumentException( + "Invalid threads value: '" + threadConfiguration + "'. Value must be positive."); + } + + return threads; + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Invalid threads value: '" + threadConfiguration + "'. Supported are integer values."); + } + } + } } diff --git a/maven-resolver-util/src/test/java/org/eclipse/aether/util/concurrency/ExecutorUtilsTest.java b/maven-resolver-util/src/test/java/org/eclipse/aether/util/concurrency/ExecutorUtilsTest.java new file mode 100644 index 00000000..ca6d684a --- /dev/null +++ b/maven-resolver-util/src/test/java/org/eclipse/aether/util/concurrency/ExecutorUtilsTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.eclipse.aether.util.concurrency; + +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.aether.DefaultRepositorySystemSession; +import org.eclipse.aether.RepositorySystemSession; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ExecutorUtilsTest { + + @Test + public void threadCountAsNumber() { + int threads = ExecutorUtils.threadCount(aSession(123), 1, "threads"); + assertEquals(123, threads); + } + + @Test + public void threadCountAsString() { + int threads = ExecutorUtils.threadCount(aSession("123"), 1, "threads"); + assertEquals(123, threads); + } + + @Test + public void dynamicThreadCount() { + int threads = ExecutorUtils.threadCount(aSession("2C"), 1, "threads"); + assertEquals(Runtime.getRuntime().availableProcessors() * 2, threads); + } + + private RepositorySystemSession aSession(Object threads) { + DefaultRepositorySystemSession session = new DefaultRepositorySystemSession(); + Map<String, Object> properties = new HashMap<>(); + properties.put("threads", threads); + session.setConfigProperties(properties); + + return session; + } +} diff --git a/src/site/markdown/configuration.md b/src/site/markdown/configuration.md index 6941e4cd..e74938c8 100644 --- a/src/site/markdown/configuration.md +++ b/src/site/markdown/configuration.md @@ -30,7 +30,7 @@ Option | Type | Description | Default Value | Supports Repo ID Suffix `aether.checksums.omitChecksumsForExtensions` | String | Comma-separated list of extensions with leading dot (example `.asc`) that should have checksums omitted. These are applied to sub-artifacts only. Note: to achieve 1.7.x `aether.checksums.forSignature=true` behaviour, pass empty string as value for this property. | `.asc` | no `aether.checksums.algorithms` | String | Comma-separated list of checksum algorithms with which checksums are validated (downloaded) and generated (uploaded). Resolver by default supports following algorithms: `MD5`, `SHA-1`, `SHA-256` and `SHA-512`. New algorithms can be added by implementing `ChecksumAlgorithmFactory` component. | `"SHA-1,MD5"` | no `aether.conflictResolver.verbose` | boolean | Flag controlling the conflict resolver's verbose mode. | `false` | no -`aether.connector.basic.threads` or `maven.artifact.threads` | int | Number of threads to use for uploading/downloading. | `5` | no +`aether.connector.basic.threads` or `maven.artifact.threads` | String | Number of threads to use for uploading/downloading. Accepts "dynamic" expressions like `1.5C` as well, or plain integer. | `5` | no `aether.connector.basic.parallelPut` | boolean | Enables or disables parallel PUT processing (parallel deploys) on basic connector globally or per remote repository. When disabled, connector behaves exactly as in Maven 3.8.x did: GETs are parallel while PUTs are sequential. | `true` | yes `aether.connector.classpath.loader` | ClassLoader | `ClassLoader` from which resources should be retrieved which start with the `classpath:` protocol. | `Thread.currentThread().getContextClassLoader()` | no `aether.connector.connectTimeout` | long | Connect timeout in milliseconds. | `10000` | yes @@ -60,7 +60,7 @@ Option | Type | Description | Default Value | Supports Repo ID Suffix `aether.dependencyCollector.maxExceptions` | int | Only exceptions up to the number given in this configuration property are emitted. Exceptions which exceed that number are swallowed. | `50` | no `aether.dependencyCollector.impl` | String | The name of the dependency collector implementation to use: depth-first (original) named `df`, and breadth-first (new in 1.8.0) named `bf`. Both collectors produce equivalent results, but they may differ performance wise, depending on project being applied to. Our experience shows that existing `df` is well suited for smaller to medium size projects, while `bf` may perform better on huge projects with many dependencies. Experiment (and come ba [...] `aether.dependencyCollector.bf.skipper` | boolean | Flag controlling whether to skip resolving duplicate/conflicting nodes during the breadth-first (`bf`) dependency collection process. | `true` | no -`aether.dependencyCollector.bf.threads` or `maven.artifact.threads` | int | Number of threads to use for collecting POMs and version ranges in BF collector. | `5` | no +`aether.dependencyCollector.bf.threads` or `maven.artifact.threads` | String | Number of threads to use for collecting POMs and version ranges in BF collector. Accepts "dynamic" expressions like `1.5C` as well, or plain integer. | `5` | no `aether.dependencyCollector.pool.artifact` | String | Flag controlling interning data pool type used by dependency collector for Artifact instances, matters for heap consumption. By default uses "weak" references (consume less heap). Using "hard" will make it much more memory aggressive and possibly faster (system and Java dependent). Supported values: `"hard"`, `"weak"`. | `"weak"` | no `aether.dependencyCollector.pool.dependency` | String | Flag controlling interning data pool type used by dependency collector for Dependency instances, matters for heap consumption. By default uses "weak" references (consume less heap). Using "hard" will make it much more memory aggressive and possibly faster (system and Java dependent). Supported values: `"hard"`, `"weak"`. | `"weak"` | no `aether.dependencyCollector.pool.descriptor` | String | Flag controlling interning data pool type used by dependency collector for Artifact Descriptor (POM) instances, matters for heap consumption. By default uses "hard" references (consume more heap, but is faster). Using "weak" will make resolver much more memory conservative, at the cost of up to 10% slower collecting dependency speed (system and Java dependent). Supported values: `"hard"`, `"weak"`. | `"hard"` | no @@ -76,7 +76,7 @@ Option | Type | Description | Default Value | Supports Repo ID Suffix `aether.enhancedLocalRepository.releasesPrefix` | String | The prefix to use for release artifacts. | `"releases"` | no `aether.enhancedLocalRepository.trackingFilename` | String | Filename of the file in which to track the remote repositories. | `"_remote.repositories"` | no `aether.interactive` | boolean | A flag indicating whether interaction with the user is allowed. | `false` | no -`aether.metadataResolver.threads` | int | Number of threads to use in parallel for resolving metadata. | `4` | no +`aether.metadataResolver.threads` | String | Number of threads to use in parallel for resolving metadata. Accepts "dynamic" expressions like `1.5C` as well, or plain integer. | `4` | no `aether.offline.protocols` | String | Comma-separated list of protocols which are supposed to be resolved offline. | - | no `aether.offline.hosts` | String | Comma-separated list of hosts which are supposed to be resolved offline. | - | no `aether.priority.<class>` | float | The priority to use for a certain extension class. `class` can either be the fully qualified name or the simple name stands for fully qualified class name. If the class name ends with `Factory` that suffix could optionally be left out. | - | no
