This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/storm.git
commit 46f85a024103f7990dd4c6a47a271f7ace8ff587 Author: Richard Zowalla <richard.zowa...@hs-heilbronn.de> AuthorDate: Wed Dec 6 16:10:54 2023 +0100 STORM-4011 - Migrate Storm itself to use Commons Collections 4 --- examples/storm-starter/pom.xml | 417 ++++++++++----------- .../starter/tools/NthLastModifiedTimeTracker.java | 38 +- pom.xml | 6 +- .../assignments/LocalAssignmentsBackendTest.java | 2 +- storm-server/pom.xml | 4 +- .../org/apache/storm/blobstore/BlobStoreUtils.java | 3 +- .../supervisor/timer/SupervisorHeartbeat.java | 2 +- storm-shaded-deps/pom.xml | 10 +- 8 files changed, 246 insertions(+), 236 deletions(-) diff --git a/examples/storm-starter/pom.xml b/examples/storm-starter/pom.xml index 9102b7279..f449979f2 100644 --- a/examples/storm-starter/pom.xml +++ b/examples/storm-starter/pom.xml @@ -15,219 +15,218 @@ See the License for the specific language governing permissions and limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>storm</artifactId> - <groupId>org.apache.storm</groupId> - <version>2.7.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>storm</artifactId> + <groupId>org.apache.storm</groupId> + <version>2.7.0-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> - <artifactId>storm-starter</artifactId> - <packaging>jar</packaging> + <artifactId>storm-starter</artifactId> + <packaging>jar</packaging> - <name>storm-starter</name> + <name>storm-starter</name> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <hbase.version>0.98.4-hadoop2</hbase.version> - </properties> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <hbase.version>0.98.4-hadoop2</hbase.version> + </properties> - <dependencies> - <dependency> - <groupId>org.hdrhistogram</groupId> - <artifactId>HdrHistogram</artifactId> - </dependency> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>6.8.5</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest</artifactId> - </dependency> - <dependency> - <groupId>org.easytesting</groupId> - <artifactId>fest-assert-core</artifactId> - <version>2.0M8</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.jmock</groupId> - <artifactId>jmock</artifactId> - <version>2.6.0</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-clojure</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-clojure-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-client</artifactId> - <version>${project.version}</version> - <!-- - Use "provided" scope to keep storm out of the jar-with-dependencies - For IntelliJ dev, intellij will load properly. - --> - <scope>${provided.scope}</scope> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-client</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>multilang-javascript</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>multilang-ruby</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>multilang-python</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-metrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-hdfs</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.storm</groupId> - <artifactId>storm-redis</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> + <dependencies> + <dependency> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> + </dependency> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <version>6.8.5</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest</artifactId> + </dependency> + <dependency> + <groupId>org.easytesting</groupId> + <artifactId>fest-assert-core</artifactId> + <version>2.0M8</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.jmock</groupId> + <artifactId>jmock</artifactId> + <version>2.6.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-clojure</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-clojure-test</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-client</artifactId> + <version>${project.version}</version> + <!-- + Use "provided" scope to keep storm out of the jar-with-dependencies + For IntelliJ dev, intellij will load properly. + --> + <scope>${provided.scope}</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-client</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>multilang-javascript</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>multilang-ruby</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>multilang-python</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-hdfs</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-redis</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> - <build> - <sourceDirectory>src/jvm</sourceDirectory> - <testSourceDirectory>test/jvm</testSourceDirectory> - <resources> - <resource> - <directory>${basedir}/multilang</directory> - </resource> - </resources> + <build> + <sourceDirectory>src/jvm</sourceDirectory> + <testSourceDirectory>test/jvm</testSourceDirectory> + <resources> + <resource> + <directory>${basedir}/multilang</directory> + </resource> + </resources> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <createDependencyReducedPom>true</createDependencyReducedPom> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.sf</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.dsa</exclude> - <exclude>META-INF/*.RSA</exclude> - <exclude>META-INF/*.rsa</exclude> - <exclude>META-INF/*.EC</exclude> - <exclude>META-INF/*.ec</exclude> - <exclude>META-INF/MSFTSIG.SF</exclude> - <exclude>META-INF/MSFTSIG.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>com.theoryinpractise</groupId> - <artifactId>clojure-maven-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <sourceDirectories> - <sourceDirectory>src/clj</sourceDirectory> - </sourceDirectories> - <testSourceDirectories> - <testSourceDirectory>test/clj</testSourceDirectory> - </testSourceDirectories> - </configuration> - <executions> - <execution> - <id>compile</id> - <phase>compile</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - <execution> - <id>test-clojure</id> - <phase>test</phase> - <goals> - <goal>test</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <!-- TestNG provider does not understand the JUnit 5 groups syntax used in the rest of the project, make sure to override it --> - <excludedGroups>none</excludedGroups> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-checkstyle-plugin</artifactId> - <!--Note - the version would be inherited--> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-pmd-plugin</artifactId> - </plugin> - </plugins> - </build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <createDependencyReducedPom>true</createDependencyReducedPom> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.sf</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.dsa</exclude> + <exclude>META-INF/*.RSA</exclude> + <exclude>META-INF/*.rsa</exclude> + <exclude>META-INF/*.EC</exclude> + <exclude>META-INF/*.ec</exclude> + <exclude>META-INF/MSFTSIG.SF</exclude> + <exclude>META-INF/MSFTSIG.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <transformers> + <transformer + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + <transformer + implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.theoryinpractise</groupId> + <artifactId>clojure-maven-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <sourceDirectories> + <sourceDirectory>src/clj</sourceDirectory> + </sourceDirectories> + <testSourceDirectories> + <testSourceDirectory>test/clj</testSourceDirectory> + </testSourceDirectories> + </configuration> + <executions> + <execution> + <id>compile</id> + <phase>compile</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + <execution> + <id>test-clojure</id> + <phase>test</phase> + <goals> + <goal>test</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <!-- TestNG provider does not understand the JUnit 5 groups syntax used in the rest of the project, make sure to override it --> + <excludedGroups>none</excludedGroups> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <!--Note - the version would be inherited--> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-pmd-plugin</artifactId> + </plugin> + </plugins> + </build> </project> diff --git a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java index d7ab06583..67260a3b5 100644 --- a/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java +++ b/examples/storm-starter/src/jvm/org/apache/storm/starter/tools/NthLastModifiedTimeTracker.java @@ -2,9 +2,9 @@ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version * 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - * + * <p> * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions * and limitations under the License. @@ -12,7 +12,8 @@ package org.apache.storm.starter.tools; -import org.apache.commons.collections.buffer.CircularFifoBuffer; +import java.util.concurrent.ArrayBlockingQueue; + import org.apache.storm.utils.Time; /** @@ -27,21 +28,21 @@ public class NthLastModifiedTimeTracker { private static final int MILLIS_IN_SEC = 1000; - private final CircularFifoBuffer lastModifiedTimesMillis; + private final ArrayBlockingQueue<Long> lastModifiedTimesMillis; public NthLastModifiedTimeTracker(int numTimesToTrack) { if (numTimesToTrack < 1) { throw new IllegalArgumentException( - "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); + "numTimesToTrack must be greater than zero (you requested " + numTimesToTrack + ")"); } - lastModifiedTimesMillis = new CircularFifoBuffer(numTimesToTrack); - initLastModifiedTimesMillis(); + lastModifiedTimesMillis = new ArrayBlockingQueue<>(numTimesToTrack); + initLastModifiedTimesMillis(numTimesToTrack); } - private void initLastModifiedTimesMillis() { + private void initLastModifiedTimesMillis(int numTimesToTrack) { long nowCached = now(); - for (int i = 0; i < lastModifiedTimesMillis.maxSize(); i++) { - lastModifiedTimesMillis.add(Long.valueOf(nowCached)); + for (int i = 0; i < numTimesToTrack; i++) { + lastModifiedTimesMillis.add(nowCached); } } @@ -50,8 +51,12 @@ public class NthLastModifiedTimeTracker { } public int secondsSinceOldestModification() { - long modifiedTimeMillis = ((Long) lastModifiedTimesMillis.get()).longValue(); - return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); + try { + long modifiedTimeMillis = lastModifiedTimesMillis.take(); + return (int) ((now() - modifiedTimeMillis) / MILLIS_IN_SEC); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } public void markAsModified() { @@ -59,7 +64,14 @@ public class NthLastModifiedTimeTracker { } private void updateLastModifiedTime() { - lastModifiedTimesMillis.add(now()); + if (!lastModifiedTimesMillis.offer(now())) { + lastModifiedTimesMillis.poll(); + try { + lastModifiedTimesMillis.put(now()); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } } } diff --git a/pom.xml b/pom.xml index 8fded073f..9f1d7ef57 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ <commons-lang.version>2.6</commons-lang.version> <commons-lang3.version>3.13.0</commons-lang3.version> <commons-exec.version>1.3</commons-exec.version> - <commons-collections.version>3.2.2</commons-collections.version> + <commons-collections.version>4.4</commons-collections.version> <commons-fileupload.version>1.5</commons-fileupload.version> <commons-codec.version>1.11</commons-codec.version> <commons-text.version>1.11.0</commons-text.version> @@ -594,8 +594,8 @@ <version>${commons-exec.version}</version> </dependency> <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> <version>${commons-collections.version}</version> </dependency> <dependency> diff --git a/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java b/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java index 2585510bd..96f5297e8 100644 --- a/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java +++ b/storm-client/test/jvm/org/apache/storm/assignments/LocalAssignmentsBackendTest.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.Set; import org.apache.storm.generated.Assignment; import org.apache.storm.generated.NodeInfo; -import org.apache.storm.shade.org.apache.commons.collections.map.HashedMap; +import org.apache.storm.shade.org.apache.commons.collections4.map.HashedMap; import org.apache.storm.utils.ConfigUtils; import org.junit.jupiter.api.Test; diff --git a/storm-server/pom.xml b/storm-server/pom.xml index 810063e37..a67a93136 100644 --- a/storm-server/pom.xml +++ b/storm-server/pom.xml @@ -115,8 +115,8 @@ <artifactId>commons-lang</artifactId> </dependency> <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> </dependency> <dependency> <groupId>net.minidev</groupId> diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java index a0644fa87..275849713 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import javax.security.auth.Subject; -import org.apache.commons.collections.CollectionUtils; import org.apache.storm.Config; import org.apache.storm.cluster.DaemonType; import org.apache.storm.generated.AuthorizationException; @@ -258,7 +257,7 @@ public class BlobStoreUtils { return; } stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key); - if (CollectionUtils.isEmpty(stateInfo)) { + if (stateInfo != null && stateInfo.isEmpty()) { return; } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java index 55a497b0b..1807e97fe 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/supervisor/timer/SupervisorHeartbeat.java @@ -19,7 +19,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.commons.collections.ListUtils; import org.apache.storm.Config; import org.apache.storm.DaemonConfig; import org.apache.storm.ServerConstants; @@ -29,6 +28,7 @@ import org.apache.storm.daemon.supervisor.SupervisorUtils; import org.apache.storm.generated.SupervisorInfo; import org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer; import org.apache.storm.scheduler.resource.normalization.NormalizedResources; +import org.apache.storm.shade.org.apache.commons.collections4.ListUtils; import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Time; import org.slf4j.Logger; diff --git a/storm-shaded-deps/pom.xml b/storm-shaded-deps/pom.xml index 80847f8d5..0ba441402 100644 --- a/storm-shaded-deps/pom.xml +++ b/storm-shaded-deps/pom.xml @@ -113,8 +113,8 @@ <optional>true</optional> </dependency> <dependency> - <groupId>commons-collections</groupId> - <artifactId>commons-collections</artifactId> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> <optional>true</optional> </dependency> <dependency> @@ -203,9 +203,9 @@ <includes> <include>com.google.guava:*</include> <include>commons-codec:commons-codec</include> - <include>commons-collections:commons-collections</include> <include>commons-io:commons-io</include> <include>commons-lang:commons-lang</include> + <include>org.apache.commons:commons-collections4</include> <!-- Pulled in by json-smart --> <include>net.minidev:*</include> <include>org.ow2.asm:*</include> @@ -255,8 +255,8 @@ <shadedPattern>org.apache.storm.shade.org.apache.commons.lang</shadedPattern> </relocation> <relocation> - <pattern>org.apache.commons.collections</pattern> - <shadedPattern>org.apache.storm.shade.org.apache.commons.collections</shadedPattern> + <pattern>org.apache.commons.collections4</pattern> + <shadedPattern>org.apache.storm.shade.org.apache.commons.collections4</shadedPattern> </relocation> <relocation> <pattern>org.jctools</pattern>