This is an automated email from the ASF dual-hosted git repository. chesnay pushed a commit to branch release-1.16 in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7ed8022da8e782d82a33b230bd6ae204f1526ea3 Author: Chesnay Schepler <ches...@apache.org> AuthorDate: Wed Sep 14 13:35:45 2022 +0200 [FLINK-29301] Automatically determine modules skipping deployment In order to work with external modules we can't rely on a static list for modules that skip deployment (== where the NOTICE contents are irrelevant). We can determine that easily from the build output. --- .../tools/ci/licensecheck/NoticeFileChecker.java | 63 +++++++++++++++++----- .../modules-skipping-deployment.modulelist | 43 --------------- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java index 8ea2cec2945..8ba7f95c449 100644 --- a/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java +++ b/tools/ci/java-ci-tools/src/main/java/org/apache/flink/tools/ci/licensecheck/NoticeFileChecker.java @@ -37,6 +37,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -51,9 +52,6 @@ public class NoticeFileChecker { private static final Logger LOG = LoggerFactory.getLogger(NoticeFileChecker.class); - private static final List<String> MODULES_SKIPPING_DEPLOYMENT = - loadFromResources("modules-skipping-deployment.modulelist"); - private static final List<String> MODULES_DEFINING_EXCESS_DEPENDENCIES = loadFromResources("modules-defining-excess-dependencies.modulelist"); @@ -64,6 +62,19 @@ public class NoticeFileChecker { private static final Pattern SHADE_INCLUDE_MODULE_PATTERN = Pattern.compile(".*Including ([^:]+):([^:]+):jar:([^ ]+) in the shaded jar"); + // Examples: + // + // Deployment on CI with alternative repo + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent --- + // [INFO] Using alternate deployment repository.../tmp/flink-validation-deployment + // + // Skipped deployment: + // [INFO] --- maven-deploy-plugin:2.8.2:deploy (default-deploy) @ flink-parent --- + // [INFO] Skipping artifact deployment + private static final Pattern DEPLOY_MODULE_PATTERN = + Pattern.compile( + ".maven-deploy-plugin:.*:deploy .* @ (?<module>[^ _]+)(_[0-9.]+)? --.*"); + // Examples: // "- org.apache.htrace:htrace-core:3.1.0-incubating" // or @@ -80,13 +91,22 @@ public class NoticeFileChecker { parseModulesFromBuildResult(buildResult), DependencyParser.parseDependencyCopyOutput(buildResult.toPath())); + final Set<String> deployedModules = parseDeployedModulesFromBuildResult(buildResult); + LOG.info( "Extracted " + + deployedModules.size() + + " modules that were deployed of which " + modulesWithBundledDependencies.keySet().size() - + " modules with a total of " + + " bundle dependencies with a total of " + modulesWithBundledDependencies.values().size() + " dependencies"); + final HashSet<String> moduleSkippingDeployment = + new HashSet<>(modulesWithBundledDependencies.keySet()); + moduleSkippingDeployment.removeAll(deployedModules); + moduleSkippingDeployment.forEach(modulesWithBundledDependencies::removeAll); + // find modules producing a shaded-jar List<Path> noticeFiles = findNoticeFiles(root); LOG.info("Found {} NOTICE files to check", noticeFiles.size()); @@ -127,15 +147,13 @@ public class NoticeFileChecker { .map(NoticeFileChecker::getModuleFromNoticeFile) .collect(Collectors.toList())); for (String moduleWithoutNotice : shadingModules) { - if (!MODULES_SKIPPING_DEPLOYMENT.contains(moduleWithoutNotice)) { - LOG.error( - "Module {} is missing a NOTICE file. It has shaded dependencies: {}", - moduleWithoutNotice, - modulesWithShadedDependencies.get(moduleWithoutNotice).stream() - .map(Dependency::toString) - .collect(Collectors.joining("\n\t", "\n\t", ""))); - severeIssueCount++; - } + LOG.error( + "Module {} is missing a NOTICE file. It has shaded dependencies: {}", + moduleWithoutNotice, + modulesWithShadedDependencies.get(moduleWithoutNotice).stream() + .map(Dependency::toString) + .collect(Collectors.joining("\n\t", "\n\t", ""))); + severeIssueCount++; } return severeIssueCount; } @@ -317,6 +335,25 @@ public class NoticeFileChecker { return result; } + private static Set<String> parseDeployedModulesFromBuildResult(File buildResult) + throws IOException { + final Set<String> deployedModules = new HashSet<>(); + try (Stream<String> linesStream = Files.lines(buildResult.toPath())) { + final Iterator<String> lines = linesStream.iterator(); + while (lines.hasNext()) { + final String line = lines.next(); + final Matcher matcher = DEPLOY_MODULE_PATTERN.matcher(line); + if (matcher.find()) { + final String module = matcher.group("module"); + if (lines.hasNext() && !lines.next().contains("Skipping artifact deployment")) { + deployedModules.add(module); + } + } + } + } + return deployedModules; + } + private static List<String> loadFromResources(String fileName) { try { try (BufferedReader bufferedReader = diff --git a/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist b/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist deleted file mode 100644 index 7b0be584fee..00000000000 --- a/tools/ci/java-ci-tools/src/main/resources/modules-skipping-deployment.modulelist +++ /dev/null @@ -1,43 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# These modules are not deployed to maven central, despite their use of the shade plugin. - -flink-examples-streaming-gcp-pubsub -flink-yarn-tests -flink-docs -flink-datastream-allround-test -flink-queryable-state-test -flink-confluent-schema-registry -flink-stream-stateful-job-upgrade-test -flink-elasticsearch7-test -flink-stream-state-ttl-test -flink-state-evolution-test -flink-elasticsearch6-test -flink-rocksdb-state-memory-control-test -flink-python-test -flink-streaming-kinesis-test -flink-tpch-test -flink-streaming-kafka-test-base -flink-heavy-deployment-stress-test -flink-high-parallelism-iterations-test -flink-end-to-end-tests-common-kafka -flink-end-to-end-tests-pulsar -flink-end-to-end-tests-elasticsearch7 -flink-end-to-end-tests-elasticsearch6 -flink-sql-gateway-test