This is an automated email from the ASF dual-hosted git repository. rzo1 pushed a commit to branch curator-decouple-step1 in repository https://gitbox.apache.org/repos/asf/storm.git
commit 5deca97924fdba1c2ae4fb7adae1d74a5861b548 Author: Richard Zowalla <[email protected]> AuthorDate: Wed Nov 18 10:28:38 2026 +0100 Remove Curator types from Storm's public API Add a Storm-owned ZkClient interface, implemented on Curator in the internal org.apache.storm.zookeeper.curator package. Curator remains a dependency; no behaviour change. --- .../apache/storm/cluster/ClusterStateListener.java | 4 + .../org/apache/storm/cluster/ConnectionState.java | 12 +- .../org/apache/storm/cluster/IStateStorage.java | 11 +- .../storm/cluster/PaceMakerStateStorage.java | 3 +- .../storm/cluster/StormClusterStateImpl.java | 5 +- .../org/apache/storm/cluster/ZKStateStorage.java | 19 +- .../topology/state/TestTransactionalState.java | 6 +- .../trident/topology/state/TransactionalState.java | 78 ++++---- .../utils/StormBoundedExponentialBackoffRetry.java | 37 +++- .../apache/storm/zookeeper/ClientZookeeper.java | 121 +++++------- .../jvm/org/apache/storm/zookeeper/ZkClient.java | 186 ++++++++++++++++++ .../zookeeper/curator/CuratorRetryPolicy.java | 44 +++++ .../{utils => zookeeper/curator}/CuratorUtils.java | 72 ++++--- .../storm/zookeeper/curator/CuratorZkClient.java | 183 ++++++++++++++++++ .../storm/zookeeper/curator/package-info.java | 28 +++ .../curator}/CuratorUtilsTest.java | 9 +- .../zookeeper/curator/CuratorZkClientTest.java | 211 +++++++++++++++++++++ .../jvm/org/apache/storm/trident/StateTest.java | 47 +++-- .../org/apache/storm/blobstore/BlobStoreUtils.java | 25 +-- .../apache/storm/blobstore/KeySequenceNumber.java | 27 ++- .../apache/storm/blobstore/LocalFsBlobStore.java | 10 +- .../blobstore/LocalFsBlobStoreSynchronizer.java | 6 +- .../org/apache/storm/daemon/nimbus/Nimbus.java | 16 +- .../storm/nimbus/LeaderListenerCallback.java | 9 +- .../org/apache/storm/zookeeper/AclEnforcement.java | 71 +++---- .../apache/storm/zookeeper/LeaderElectorImp.java | 38 +++- .../org/apache/storm/zookeeper/LeaderListener.java | 37 ++++ .../zookeeper/LeaderListenerCallbackFactory.java | 10 +- .../java/org/apache/storm/zookeeper/Zookeeper.java | 21 +- .../storm/blobstore/KeySequenceNumberTest.java | 3 +- .../LocalFsBlobStoreSynchronizerTest.java | 3 +- .../blobstore/MockZookeeperClientBuilder.java | 57 +++--- .../org/apache/storm/cluster/ClusterStateTest.java | 25 +-- .../apache/storm/zookeeper/AclEnforcementTest.java | 188 ++++++++++++++++++ 34 files changed, 1280 insertions(+), 342 deletions(-) diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java index f0968a14d5..dd4ad6209b 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ClusterStateListener.java @@ -12,6 +12,10 @@ package org.apache.storm.cluster; +/** + * Listener for connection state changes of the client backing an {@link IStateStorage}. + */ +@FunctionalInterface public interface ClusterStateListener { void stateChanged(ConnectionState newState); } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java b/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java index 0fa75382d7..43a00b5118 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ConnectionState.java @@ -12,8 +12,18 @@ package org.apache.storm.cluster; +/** + * Connection state of a ZooKeeper client, as delivered to a {@link ClusterStateListener}. + */ public enum ConnectionState { + /** The first successful connection. */ CONNECTED, + /** The connection was re-established after being suspended or lost. */ RECONNECTED, - LOST + /** The session is considered lost (expired or the connection timed out). */ + LOST, + /** The connection was interrupted (ZooKeeper Disconnected); the session may still be alive. */ + SUSPENDED, + /** Connected in read-only mode (ZooKeeper ConnectedReadOnly). */ + READ_ONLY } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java index abc35b0cf9..48bffcd17f 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/IStateStorage.java @@ -15,7 +15,6 @@ package org.apache.storm.cluster; import java.io.Closeable; import java.util.List; import org.apache.storm.callback.ZKStateChangedCallback; -import org.apache.storm.shade.org.apache.curator.framework.state.ConnectionStateListener; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; /** @@ -206,11 +205,15 @@ public interface IStateStorage extends Closeable { void delete_worker_hb(String path); /** - * Add a StateStorageListener to the connection. + * Add a listener for connection state changes of the underlying store. * - * @param listener A StateStorageListener to handle changing cluster state events. + * <p>This takes Storm's own {@link ClusterStateListener}. Earlier releases took Curator's + * {@code ConnectionStateListener}; implementations of {@code storm.cluster.state.store} must + * be updated accordingly. + * + * @param listener A ClusterStateListener to handle changing cluster state events. */ - void add_listener(ConnectionStateListener listener); + void add_listener(ClusterStateListener listener); /** * Force consistency on a path. Any writes committed on the path before this call will be diff --git a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java index ec8b061c8c..d7ea0328e5 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/PaceMakerStateStorage.java @@ -30,7 +30,6 @@ import org.apache.storm.generated.HBPulse; import org.apache.storm.generated.HBServerMessageType; import org.apache.storm.pacemaker.PacemakerClientPool; import org.apache.storm.pacemaker.PacemakerConnectionException; -import org.apache.storm.shade.org.apache.curator.framework.state.ConnectionStateListener; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.utils.Utils; import org.apache.storm.utils.WrappedHBExecutionException; @@ -286,7 +285,7 @@ public class PaceMakerStateStorage implements IStateStorage { } @Override - public void add_listener(ConnectionStateListener listener) { + public void add_listener(ClusterStateListener listener) { stateStorage.add_listener(listener); } diff --git a/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java b/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java index f6610eca5b..286779dd0b 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/StormClusterStateImpl.java @@ -41,7 +41,6 @@ import org.apache.storm.generated.SupervisorInfo; import org.apache.storm.generated.WorkerTokenServiceType; import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.shade.org.apache.commons.lang3.StringUtils; -import org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.Watcher; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; @@ -296,10 +295,10 @@ public class StormClusterStateImpl implements IStormClusterState { public void addNimbusHost(final String nimbusId, final NimbusSummary nimbusSummary) { // explicit delete for ephmeral node to ensure this session creates the entry. stateStorage.delete_node(ClusterUtils.nimbusPath(nimbusId)); - stateStorage.add_listener((curatorFramework, connectionState) -> { + stateStorage.add_listener(connectionState -> { LOG.info("Connection state listener invoked, zookeeper connection state has changed " + "to {}", connectionState); - if (connectionState.equals(ConnectionState.RECONNECTED)) { + if (connectionState == ConnectionState.RECONNECTED) { LOG.info("Connection state has changed to reconnected so setting nimbuses entry " + "one more time"); // explicit delete for ephemeral node to ensure this session creates the entry. diff --git a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java index 102702fd03..53d2011efe 100644 --- a/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java +++ b/storm-client/src/jvm/org/apache/storm/cluster/ZKStateStorage.java @@ -27,14 +27,13 @@ import org.apache.storm.Config; import org.apache.storm.callback.DefaultWatcherCallBack; import org.apache.storm.callback.WatcherCallBack; import org.apache.storm.callback.ZKStateChangedCallback; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.state.ConnectionStateListener; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.Watcher; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.utils.Utils; import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +44,8 @@ public class ZKStateStorage implements IStateStorage { private ConcurrentHashMap<String, ZKStateChangedCallback> callbacks = new ConcurrentHashMap<String, ZKStateChangedCallback>(); - private CuratorFramework zkWriter; - private CuratorFramework zkReader; + private ZkClient zkWriter; + private ZkClient zkReader; private AtomicBoolean active; private boolean isNimbus; @@ -62,7 +61,7 @@ public class ZKStateStorage implements IStateStorage { } // just mkdir STORM_ZOOKEEPER_ROOT dir - CuratorFramework zkTemp = mkZk(context.getDaemonType()); + ZkClient zkTemp = mkZk(context.getDaemonType()); String rootPath = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT)); ClientZookeeper.mkdirs(zkTemp, rootPath, context.getDefaultZkAcls()); zkTemp.close(); @@ -78,14 +77,14 @@ public class ZKStateStorage implements IStateStorage { } @SuppressWarnings("unchecked") - private CuratorFramework mkZk(DaemonType type) { + private ZkClient mkZk(DaemonType type) { return ClientZookeeper.mkClient(conf, (List<String>) conf .get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT), "", new DefaultWatcherCallBack(), authConf, type); } @SuppressWarnings("unchecked") - private CuratorFramework mkZk(WatcherCallBack watcher, + private ZkClient mkZk(WatcherCallBack watcher, DaemonType type) throws NumberFormatException { return ClientZookeeper.mkClient(conf, (List<String>) conf .get(Config.STORM_ZOOKEEPER_SERVERS), @@ -225,10 +224,8 @@ public class ZKStateStorage implements IStateStorage { } @Override - public void add_listener(final ConnectionStateListener listener) { - ClientZookeeper.addListener(zkReader, - (curatorFramework, connectionState) -> listener.stateChanged(curatorFramework, - connectionState)); + public void add_listener(ClusterStateListener listener) { + ClientZookeeper.addListener(zkReader, listener); } @Override diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java index cae188d325..f212484db8 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TestTransactionalState.java @@ -14,9 +14,9 @@ package org.apache.storm.trident.topology.state; import java.util.List; import java.util.Map; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.zookeeper.ZkClient; /** * Facilitates testing of non-public methods in the parent class. @@ -30,9 +30,9 @@ public class TestTransactionalState extends TransactionalState { super(conf, id, subroot); } - public static void createNode(CuratorFramework curator, + public static void createNode(ZkClient zk, String rootDir, byte[] data, List<ACL> acls, CreateMode mode) throws Exception { - TransactionalState.createNode(curator, rootDir, data, acls, mode); + TransactionalState.createNode(zk, rootDir, data, acls, mode); } } diff --git a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java index c98028d842..e66e32883c 100644 --- a/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java +++ b/storm-client/src/jvm/org/apache/storm/trident/topology/state/TransactionalState.java @@ -21,27 +21,25 @@ import org.apache.storm.Config; import org.apache.storm.cluster.DaemonType; import org.apache.storm.shade.net.minidev.json.JSONValue; import org.apache.storm.shade.net.minidev.json.parser.ParseException; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.api.PathAndBytesable; -import org.apache.storm.shade.org.apache.curator.framework.api.ProtectACLCreateModePathAndBytesable; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; -import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.Utils; -import org.apache.storm.utils.ZookeeperAuthInfo; +import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Class that contains the logic to extract the transactional state info from zookeeper. All * transactional state is kept in zookeeper. This - * class only contains references to Curator, which is used to get all info from zookeeper. + * class only contains references to a {@link ZkClient}, which is used to get all info from + * zookeeper. */ public class TransactionalState { private static final Logger LOG = LoggerFactory.getLogger(TransactionalState.class); - CuratorFramework curator; + ZkClient zkClient; List<ACL> zkAcls = null; protected TransactionalState(Map<String, Object> conf, String id, String subroot) { @@ -54,9 +52,8 @@ public class TransactionalState { Config.STORM_ZOOKEEPER_SERVERS); Object port = getWithBackup(conf, Config.TRANSACTIONAL_ZOOKEEPER_PORT, Config.STORM_ZOOKEEPER_PORT); - ZookeeperAuthInfo auth = new ZookeeperAuthInfo(conf); - CuratorFramework initter = CuratorUtils.newCuratorStarted(conf, servers, port, auth, - DaemonType.WORKER.getDefaultZkAcls(conf)); + ZkClient initter = ClientZookeeper.mkClient(conf, servers, port, "", null, conf, + DaemonType.WORKER); zkAcls = Utils.getWorkerACL(conf); try { TransactionalState.createNode(initter, transactionalRoot, null, null, null); @@ -70,8 +67,8 @@ public class TransactionalState { } initter.close(); - curator = CuratorUtils.newCuratorStarted(conf, servers, port, rootDir, auth, - DaemonType.WORKER.getDefaultZkAcls(conf)); + zkClient = ClientZookeeper.mkClient(conf, servers, port, rootDir, null, conf, + DaemonType.WORKER); } catch (Exception e) { throw new RuntimeException(e); } @@ -85,30 +82,31 @@ public class TransactionalState { return new TransactionalState(conf, id, "coordinator"); } - protected static String forPath(PathAndBytesable<String> builder, - String path, byte[] data) throws Exception { - return (data == null) - ? builder.forPath(path) - : builder.forPath(path, data); - } - - protected static void createNode(CuratorFramework curator, String path, + /** + * Create a node, and any missing parents. + * + * <p>{@code mode} is only honoured when {@code acls} is {@code null}; when {@code acls} is + * given, or {@code mode} is {@code null}, the node is created {@link CreateMode#PERSISTENT}. + * When {@code data} is {@code null} the client's default payload is stored. + * + * @param zk the client to use + * @param path the path of the node + * @param data the data to store, or {@code null} for the client's default payload + * @param acls the ACL of the node, or {@code null} for the client's default ACL + * @param mode the create mode, see above + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + protected static void createNode(ZkClient zk, String path, byte[] data, List<ACL> acls, CreateMode mode) throws Exception { - ProtectACLCreateModePathAndBytesable<String> builder = - curator.create().creatingParentsIfNeeded(); LOG.debug("Creating node [path = {}], [data = {}], [acls = {}], [mode = {}]", path, asString(data), acls, mode); - if (acls == null) { - if (mode == null) { - TransactionalState.forPath(builder, path, data); - } else { - TransactionalState.forPath(builder.withMode(mode), path, data); - } - return; + CreateMode effectiveMode = (acls == null && mode != null) ? mode : CreateMode.PERSISTENT; + if (data == null) { + zk.createWithDefaultData(path, effectiveMode, acls, true); + } else { + zk.create(path, data, effectiveMode, acls, true); } - - TransactionalState.forPath(builder.withACL(acls), path, data); } private static String asString(byte[] data) { @@ -124,10 +122,10 @@ public class TransactionalState { throw new RuntimeException(e); } try { - if (curator.checkExists().forPath(path) != null) { - curator.setData().forPath(path, ser); + if (zkClient.exists(path, false) != null) { + zkClient.setData(path, ser); } else { - TransactionalState.createNode(curator, path, ser, zkAcls, + TransactionalState.createNode(zkClient, path, ser, zkAcls, CreateMode.PERSISTENT); } } catch (KeeperException.NodeExistsException nne) { @@ -140,7 +138,7 @@ public class TransactionalState { public void delete(String path) { path = "/" + path; try { - curator.delete().forPath(path); + zkClient.delete(path, false); } catch (KeeperException.NoNodeException nne) { LOG.warn("Path {} already deleted.", path); } catch (Exception e) { @@ -153,10 +151,10 @@ public class TransactionalState { path = "/" + path; try { List<String> children; - if (curator.checkExists().forPath(path) == null) { + if (zkClient.exists(path, false) == null) { children = new ArrayList<>(); } else { - children = curator.getChildren().forPath(path); + children = zkClient.getChildren(path, false); } LOG.debug("List [path = {}], [children = {}]", path, children); return children; @@ -173,12 +171,12 @@ public class TransactionalState { path = "/" + path; try { Object data; - if (curator.checkExists().forPath(path) != null) { + if (zkClient.exists(path, false) != null) { // Use parseWithException instead of parse so we can capture deserialization errors // in the log. // They are likely to be bugs in the spout code. try { - data = JSONValue.parseWithException(new String(curator.getData().forPath(path), + data = JSONValue.parseWithException(new String(zkClient.getData(path, false, null), "UTF-8")); } catch (ParseException e) { LOG.warn("Failed to deserialize zookeeper data for path {}", path, e); @@ -195,7 +193,7 @@ public class TransactionalState { } public void close() { - curator.close(); + zkClient.close(); } private Object getWithBackup(Map<String, Object> amap, String primary, String backup) { diff --git a/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java b/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java index 47790b8d94..ee5c739d65 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java +++ b/storm-client/src/jvm/org/apache/storm/utils/StormBoundedExponentialBackoffRetry.java @@ -13,14 +13,22 @@ package org.apache.storm.utils; import java.util.Random; -import org.apache.storm.shade.org.apache.curator.retry.BoundedExponentialBackoffRetry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StormBoundedExponentialBackoffRetry extends BoundedExponentialBackoffRetry { +/** + * Exponential-then-linear backoff calculator with jitter, used for ZooKeeper, Netty, Thrift and + * Pacemaker reconnects. + * + * <p>This class does not depend on any ZooKeeper client library. In earlier releases it extended + * Curator's {@code BoundedExponentialBackoffRetry}; it no longer is a Curator {@code RetryPolicy}. + */ +public class StormBoundedExponentialBackoffRetry { private static final Logger LOG = LoggerFactory .getLogger(StormBoundedExponentialBackoffRetry.class); private final Random random = new Random(); + private final int baseSleepTimeMs; + private final int maxSleepTimeMs; private final int linearBaseSleepMs; private final int stepSize; private int expRetriesThreshold; @@ -36,7 +44,8 @@ public class StormBoundedExponentialBackoffRetry extends BoundedExponentialBacko */ public StormBoundedExponentialBackoffRetry(int baseSleepTimeMs, int maxSleepTimeMs, int maxRetries) { - super(baseSleepTimeMs, maxSleepTimeMs, maxRetries); + this.baseSleepTimeMs = baseSleepTimeMs; + this.maxSleepTimeMs = maxSleepTimeMs; expRetriesThreshold = 1; while ((1 << (expRetriesThreshold + 1)) < ((maxSleepTimeMs - baseSleepTimeMs) / 2)) { expRetriesThreshold++; @@ -54,20 +63,34 @@ public class StormBoundedExponentialBackoffRetry extends BoundedExponentialBacko } else { this.stepSize = 1; } - this.linearBaseSleepMs = super.getBaseSleepTimeMs() + (1 << expRetriesThreshold); + this.linearBaseSleepMs = this.baseSleepTimeMs + (1 << expRetriesThreshold); + } + + public int getBaseSleepTimeMs() { + return baseSleepTimeMs; } - @Override + public int getMaxSleepTimeMs() { + return maxSleepTimeMs; + } + + /** + * Compute the time to sleep before the given retry. + * + * @param retryCount the number of retries already attempted (0 based) + * @param elapsedTimeMs elapsed time since the first attempt (unused) + * @return the time to sleep in milliseconds + */ public long getSleepTimeMs(int retryCount, long elapsedTimeMs) { if (retryCount < expRetriesThreshold) { int exp = 1 << retryCount; int jitter = random.nextInt(exp); - long sleepTimeMs = super.getBaseSleepTimeMs() + exp + jitter; + long sleepTimeMs = baseSleepTimeMs + exp + jitter; LOG.debug("WILL SLEEP FOR {}ms (NOT MAX)", sleepTimeMs); return sleepTimeMs; } else { int stepJitter = random.nextInt(stepSize); - long sleepTimeMs = Math.min(super.getMaxSleepTimeMs(), (linearBaseSleepMs + long sleepTimeMs = Math.min(maxSleepTimeMs, (linearBaseSleepMs + (stepSize * (retryCount - expRetriesThreshold)) + stepJitter)); LOG.debug("WILL SLEEP FOR {}ms (MAX)", sleepTimeMs); return sleepTimeMs; diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java b/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java index be53a8aa23..a3ce057924 100644 --- a/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/ClientZookeeper.java @@ -22,19 +22,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.storm.callback.WatcherCallBack; +import org.apache.storm.cluster.ClusterStateListener; import org.apache.storm.cluster.DaemonType; import org.apache.storm.cluster.VersionedData; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.api.CuratorEventType; -import org.apache.storm.shade.org.apache.curator.framework.state.ConnectionStateListener; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; -import org.apache.storm.shade.org.apache.zookeeper.WatchedEvent; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.shade.org.apache.zookeeper.data.Stat; -import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.Utils; import org.apache.storm.utils.ZookeeperAuthInfo; +import org.apache.storm.zookeeper.curator.CuratorUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,11 +63,24 @@ public class ClientZookeeper { _instance = INSTANCE; } - public static void mkdirs(CuratorFramework zk, String path, List<ACL> acls) { + public static void mkdirs(ZkClient zk, String path, List<ACL> acls) { _instance.mkdirsImpl(zk, path, acls); } - public static CuratorFramework mkClient(Map<String, Object> conf, List<String> servers, + /** + * Create and start a ZooKeeper client. + * + * @param conf the Storm configuration (timeouts, retries, TLS) + * @param servers the ZooKeeper servers + * @param port the ZooKeeper port + * @param root the chroot, {@code ""} for none + * @param watcher receives events of watches set through the client; may be {@code null}, in + * which case watch events are dropped + * @param authConf configuration holding the ZooKeeper authentication, may be {@code null} + * @param type the daemon type, which determines the client's default ACL + * @return the started client + */ + public static ZkClient mkClient(Map<String, Object> conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map<String, Object> authConf, DaemonType type) { return _instance.mkClientImpl(conf, servers, port, root, watcher, authConf, type); @@ -78,7 +88,7 @@ public class ClientZookeeper { // Deletes the state inside the zookeeper for a key, for which the // contents of the key starts with nimbus host port information - public static void deleteNodeBlobstore(CuratorFramework zk, String parentPath, + public static void deleteNodeBlobstore(ZkClient zk, String parentPath, String hostPortInfo) { String normalizedParentPath = normalizePath(parentPath); List<String> childPathList = null; @@ -93,20 +103,19 @@ public class ClientZookeeper { } } - public static String createNode(CuratorFramework zk, String path, byte[] data, CreateMode mode, + public static String createNode(ZkClient zk, String path, byte[] data, CreateMode mode, List<ACL> acls) { String ret = null; try { String npath = normalizePath(path); - ret = zk.create().creatingParentsIfNeeded().withMode(mode).withACL(acls).forPath(npath, - data); + ret = zk.create(npath, data, mode, acls, true); } catch (Exception e) { throw Utils.wrapInRuntime(e); } return ret; } - public static String createNode(CuratorFramework zk, String path, byte[] data, List<ACL> acls) { + public static String createNode(ZkClient zk, String path, byte[] data, List<ACL> acls) { return createNode(zk, path, data, CreateMode.PERSISTENT, acls); } @@ -139,25 +148,21 @@ public class ClientZookeeper { return rtn; } - public static boolean existsNode(CuratorFramework zk, String path, boolean watch) { + public static boolean existsNode(ZkClient zk, String path, boolean watch) { Stat stat = null; try { - if (watch) { - stat = zk.checkExists().watched().forPath(normalizePath(path)); - } else { - stat = zk.checkExists().forPath(normalizePath(path)); - } + stat = zk.exists(normalizePath(path), watch); } catch (Exception e) { throw Utils.wrapInRuntime(e); } return stat != null; } - public static void deleteNode(CuratorFramework zk, String path) { + public static void deleteNode(ZkClient zk, String path) { try { String npath = normalizePath(path); if (existsNode(zk, npath, false)) { - zk.delete().deletingChildrenIfNeeded().forPath(normalizePath(path)); + zk.delete(normalizePath(path), true); } } catch (Exception e) { if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { @@ -178,54 +183,41 @@ public class ClientZookeeper { return toksToPath(toks); } - public static boolean exists(CuratorFramework zk, String path, boolean watch) { + public static boolean exists(ZkClient zk, String path, boolean watch) { return existsNode(zk, path, watch); } - public static Stat setData(CuratorFramework zk, String path, byte[] data) { + public static Stat setData(ZkClient zk, String path, byte[] data) { try { String npath = normalizePath(path); - return zk.setData().forPath(npath, data); + return zk.setData(npath, data); } catch (Exception e) { throw Utils.wrapInRuntime(e); } } - public static Integer getVersion(CuratorFramework zk, String path, + public static Integer getVersion(ZkClient zk, String path, boolean watch) throws Exception { String npath = normalizePath(path); - // checkExists returns the Stat directly (null when absent) and still arms the watch when - // watch is true. request 1. - Stat stat; - if (watch) { - stat = zk.checkExists().watched().forPath(npath); - } else { - stat = zk.checkExists().forPath(npath); // request 2. - } + // exists returns the Stat directly (null when absent) and still arms the watch when + // watch is true. + Stat stat = zk.exists(npath, watch); return stat == null ? null : Integer.valueOf(stat.getVersion()); } - public static List<String> getChildren(CuratorFramework zk, String path, boolean watch) { + public static List<String> getChildren(ZkClient zk, String path, boolean watch) { try { String npath = normalizePath(path); - if (watch) { - return zk.getChildren().watched().forPath(npath); - } else { - return zk.getChildren().forPath(npath); - } + return zk.getChildren(npath, watch); } catch (Exception e) { throw Utils.wrapInRuntime(e); } } - public static byte[] getData(CuratorFramework zk, String path, boolean watch) { + public static byte[] getData(ZkClient zk, String path, boolean watch) { String npath = normalizePath(path); try { - if (watch) { - return zk.getData().watched().forPath(npath); - } else { - return zk.getData().forPath(npath); - } + return zk.getData(npath, watch, null); } catch (Exception e) { if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { // Node is absent. A watcher can notify when the node is announced. @@ -246,18 +238,13 @@ public class ClientZookeeper { * @param watch should a watch be enabled * @return null if no data is found, else the data with the version. */ - public static VersionedData<byte[]> getDataWithVersion(CuratorFramework zk, String path, + public static VersionedData<byte[]> getDataWithVersion(ZkClient zk, String path, boolean watch) { VersionedData<byte[]> data = null; String npath = normalizePath(path); Stat stats = new Stat(); try { - byte[] bytes; - if (watch) { - bytes = zk.getData().storingStatIn(stats).watched().forPath(npath); - } else { - bytes = zk.getData().storingStatIn(stats).forPath(npath); - } + byte[] bytes = zk.getData(npath, watch, stats); if (bytes != null) { data = new VersionedData<>(stats.getVersion(), bytes); } @@ -275,19 +262,19 @@ public class ClientZookeeper { return data; } - public static void addListener(CuratorFramework zk, ConnectionStateListener listener) { - zk.getConnectionStateListenable().addListener(listener); + public static void addListener(ZkClient zk, ClusterStateListener listener) { + zk.addConnectionStateListener(listener); } - public static void syncPath(CuratorFramework zk, String path) { + public static void syncPath(ZkClient zk, String path) { try { - zk.sync().forPath(normalizePath(path)); + zk.sync(normalizePath(path)); } catch (Exception e) { throw Utils.wrapInRuntime(e); } } - public void mkdirsImpl(CuratorFramework zk, String path, List<ACL> acls) { + public void mkdirsImpl(ZkClient zk, String path, List<ACL> acls) { String npath = ClientZookeeper.normalizePath(path); if (npath.equals("/")) { return; @@ -306,26 +293,12 @@ public class ClientZookeeper { } } - public CuratorFramework mkClientImpl(Map<String, Object> conf, List<String> servers, + public ZkClient mkClientImpl(Map<String, Object> conf, List<String> servers, Object port, String root, final WatcherCallBack watcher, Map<String, Object> authConf, DaemonType type) { - CuratorFramework fk; - if (authConf != null) { - fk = CuratorUtils.newCurator(conf, servers, port, root, new ZookeeperAuthInfo(authConf), - type.getDefaultZkAcls(conf)); - } else { - fk = CuratorUtils.newCurator(conf, servers, port, root, null, type - .getDefaultZkAcls(conf)); - } - - fk.getCuratorListenable().addListener((unused, e) -> { - if (e.getType().equals(CuratorEventType.WATCHED)) { - WatchedEvent event = e.getWatchedEvent(); - watcher.execute(event.getState(), event.getType(), event.getPath()); - } - }); + ZookeeperAuthInfo auth = authConf != null ? new ZookeeperAuthInfo(authConf) : null; LOG.info("Starting ZK Curator"); - fk.start(); - return fk; + return CuratorUtils.newClient(conf, servers, port, root, auth, type.getDefaultZkAcls(conf), + watcher); } } diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/ZkClient.java b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkClient.java new file mode 100644 index 0000000000..268016e77f --- /dev/null +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/ZkClient.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper; + +import java.util.List; +import org.apache.storm.cluster.ClusterStateListener; +import org.apache.storm.shade.org.apache.zookeeper.CreateMode; +import org.apache.storm.shade.org.apache.zookeeper.KeeperException; +import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.shade.org.apache.zookeeper.data.Stat; + +/** + * A Storm-owned handle to a started ZooKeeper session. + * + * <p>This is the type Storm uses in its public and protected APIs wherever a ZooKeeper client is + * passed around. It deliberately exposes only the operations Storm itself needs, and each method is + * named after the native ZooKeeper call it maps to, so the implementation can be swapped without + * touching callers. Instances are obtained from + * {@link ClientZookeeper#mkClient(java.util.Map, List, Object, String, + * org.apache.storm.callback.WatcherCallBack, java.util.Map, org.apache.storm.cluster.DaemonType)}. + * + * <p>Contract that every implementation must honour: + * <ol> + * <li><b>Paths</b> are absolute and relative to the chroot the client was created with. The + * client does not normalize them; callers such as {@link ClientZookeeper} do.</li> + * <li><b>Exceptions</b>: ZooKeeper {@link KeeperException} subclasses (for example + * {@link KeeperException.NoNodeException} and {@link KeeperException.NodeExistsException}) + * propagate unwrapped, so callers may catch them directly.</li> + * <li><b>Watches</b>: {@code watch == true} registers a one-shot ZooKeeper watch whose events + * are delivered to the {@link org.apache.storm.callback.WatcherCallBack} the client was created + * with. If the client was created without a callback, events are dropped.</li> + * <li><b>ACLs</b>: {@code acls == null} means the client's default ACL, which is the + * {@link org.apache.storm.cluster.DaemonType} default ACL given at creation, or + * {@code OPEN_ACL_UNSAFE} when that is {@code null}. Parent nodes created because + * {@code createParents == true} always get the client's default ACL, never the ACL passed to + * the call.</li> + * <li><b>Retries</b>: operations block until the session is connected and are retried on + * connection loss according to the configured + * {@link org.apache.storm.utils.StormBoundedExponentialBackoffRetry} parameters + * ({@code storm.zookeeper.retry.interval}, + * {@code storm.zookeeper.retry.intervalceiling.millis}, + * {@code storm.zookeeper.retry.times}).</li> + * <li><b>Sync</b>: {@link #sync(String)} is asynchronous (fire-and-forget) and does not wait + * for the sync to complete.</li> + * <li><b>Default data</b>: {@link #createWithDefaultData(String, CreateMode, List, boolean)} + * writes the client's default payload, which is the bytes of + * {@code InetAddress.getLocalHost().getHostAddress()}. This is distinct from + * {@link #create(String, byte[], CreateMode, List, boolean)} with {@code data == null}, which + * stores no data.</li> + * </ol> + */ +public interface ZkClient extends AutoCloseable { + + /** + * Create a node. + * + * @param path absolute path of the node to create + * @param data the data to store, may be {@code null} for no data + * @param mode the create mode + * @param acls the ACL for the node, or {@code null} for the client default ACL + * @param createParents whether missing parent nodes are created (with the client default ACL) + * @return the actual path of the created node (differs from {@code path} for sequential modes) + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + String create(String path, byte[] data, CreateMode mode, List<ACL> acls, boolean createParents) + throws Exception; + + /** + * Create a node storing the client's default payload (the local host address bytes). + * + * @param path absolute path of the node to create + * @param mode the create mode + * @param acls the ACL for the node, or {@code null} for the client default ACL + * @param createParents whether missing parent nodes are created (with the client default ACL) + * @return the actual path of the created node + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + String createWithDefaultData(String path, CreateMode mode, List<ACL> acls, + boolean createParents) throws Exception; + + /** + * Check whether a node exists. + * + * @param path absolute path + * @param watch whether to leave a watch on the node (set even when the node is absent) + * @return the node's stat, or {@code null} if the node does not exist + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + Stat exists(String path, boolean watch) throws Exception; + + /** + * Read the data of a node. + * + * @param path absolute path + * @param watch whether to leave a watch on the node + * @param stat if not {@code null}, filled with the node's stat + * @return the node data + * @throws Exception on failure, for example {@link KeeperException.NoNodeException} + */ + byte[] getData(String path, boolean watch, Stat stat) throws Exception; + + /** + * Unconditionally (any version) set the data of a node. + * + * @param path absolute path + * @param data new data + * @return the node's new stat + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + Stat setData(String path, byte[] data) throws Exception; + + /** + * List the children of a node. + * + * @param path absolute path + * @param watch whether to leave a watch on the node + * @return the child names (not full paths) + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + List<String> getChildren(String path, boolean watch) throws Exception; + + /** + * Unconditionally (any version) delete a node. + * + * @param path absolute path + * @param recursive whether to delete the node's children first + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + void delete(String path, boolean recursive) throws Exception; + + /** + * Read the ACL of a node. + * + * @param path absolute path + * @return the node's ACL + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + List<ACL> getAcl(String path) throws Exception; + + /** + * Unconditionally (any version) set the ACL of a node. + * + * @param path absolute path + * @param acls the new ACL + * @throws Exception on failure, ZooKeeper exceptions unwrapped + */ + void setAcl(String path, List<ACL> acls) throws Exception; + + /** + * Asynchronously ask the server to sync the given path with the leader. This call does not + * wait for the sync to complete. + * + * @param path absolute path + * @throws Exception if the request could not be issued + */ + void sync(String path) throws Exception; + + /** + * Register a listener for connection state changes of this client. + * + * @param listener the listener to notify + */ + void addConnectionStateListener(ClusterStateListener listener); + + /** + * Close the session and release all resources. + */ + @Override + void close(); +} diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorRetryPolicy.java b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorRetryPolicy.java new file mode 100644 index 0000000000..165898316e --- /dev/null +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorRetryPolicy.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper.curator; + +import org.apache.storm.shade.org.apache.curator.retry.BoundedExponentialBackoffRetry; +import org.apache.storm.utils.StormBoundedExponentialBackoffRetry; + +/** + * Internal. Adapts {@link StormBoundedExponentialBackoffRetry} to Curator's retry policy. + * + * <p>Extending {@link BoundedExponentialBackoffRetry} with the same arguments keeps Curator's own + * handling (the maximum-retries cap and {@code allowRetry} semantics), while the sleep time is + * computed by Storm's backoff algorithm. + */ +final class CuratorRetryPolicy extends BoundedExponentialBackoffRetry { + private final StormBoundedExponentialBackoffRetry delegate; + + CuratorRetryPolicy(int baseSleepTimeMs, int maxSleepTimeMs, int maxRetries) { + super(baseSleepTimeMs, maxSleepTimeMs, maxRetries); + this.delegate = new StormBoundedExponentialBackoffRetry(baseSleepTimeMs, maxSleepTimeMs, + maxRetries); + } + + @Override + public long getSleepTimeMs(int retryCount, long elapsedTimeMs) { + return delegate.getSleepTimeMs(retryCount, elapsedTimeMs); + } +} diff --git a/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorUtils.java similarity index 83% rename from storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java rename to storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorUtils.java index 4faf4ce095..98348ea3d2 100644 --- a/storm-client/src/jvm/org/apache/storm/utils/CuratorUtils.java +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorUtils.java @@ -16,38 +16,74 @@ * limitations under the License. */ -package org.apache.storm.utils; +package org.apache.storm.zookeeper.curator; import java.util.ArrayList; import java.util.List; import java.util.Map; import javax.naming.ConfigurationException; import org.apache.storm.Config; +import org.apache.storm.callback.WatcherCallBack; import org.apache.storm.shade.org.apache.commons.lang3.StringUtils; import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.storm.shade.org.apache.curator.framework.api.ACLProvider; +import org.apache.storm.shade.org.apache.curator.framework.api.CuratorEventType; +import org.apache.storm.shade.org.apache.zookeeper.WatchedEvent; import org.apache.storm.shade.org.apache.zookeeper.client.ZKClientConfig; import org.apache.storm.shade.org.apache.zookeeper.common.ClientX509Util; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.ZookeeperAuthInfo; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CuratorUtils { +/** + * Internal. Builds Curator clients from a Storm configuration. + * + * <p>Not part of Storm's public API; may change or be removed without notice. Use + * {@link org.apache.storm.zookeeper.ClientZookeeper#mkClient} to obtain a {@link ZkClient}. + */ +public final class CuratorUtils { public static final Logger LOG = LoggerFactory.getLogger(CuratorUtils.class); public static final String CLIENT_CNXN = org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNetty.class.getName(); - public static CuratorFramework newCurator(Map<String, Object> conf, List<String> servers, - Object port, String root, - List<ACL> defaultAcl) { - return newCurator(conf, servers, port, root, null, defaultAcl); + private CuratorUtils() { + } + + /** + * Build, start and wrap a Curator client. + * + * @param conf the Storm configuration + * @param servers the ZooKeeper servers + * @param port the ZooKeeper port + * @param root the chroot, {@code ""} for none + * @param auth the authentication to add, may be {@code null} + * @param defaultAcl the default ACL for created nodes, {@code null} for OPEN_ACL_UNSAFE + * @param watcher receives watch events, {@code null} to drop them + * @return the started client + */ + public static ZkClient newClient(Map<String, Object> conf, List<String> servers, Object port, + String root, ZookeeperAuthInfo auth, List<ACL> defaultAcl, WatcherCallBack watcher) { + CuratorFramework fk = newCurator(conf, servers, port, root, auth, defaultAcl); + if (watcher != null) { + fk.getCuratorListenable().addListener((unused, e) -> { + if (e.getType().equals(CuratorEventType.WATCHED)) { + WatchedEvent event = e.getWatchedEvent(); + watcher.execute(event.getState(), event.getType(), event.getPath()); + } + }); + } + fk.start(); + return new CuratorZkClient(fk); } public static CuratorFramework newCurator(Map<String, Object> conf, List<String> servers, - Object port, ZookeeperAuthInfo auth, + Object port, String root, List<ACL> defaultAcl) { - return newCurator(conf, servers, port, "", auth, defaultAcl); + return newCurator(conf, servers, port, root, null, defaultAcl); } public static CuratorFramework newCurator(Map<String, Object> conf, List<String> servers, @@ -87,7 +123,7 @@ public class CuratorUtils { .get(Config.STORM_ZOOKEEPER_CONNECTION_TIMEOUT))) .sessionTimeoutMs(ObjectReader.getInt(conf .get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT))) - .retryPolicy(new StormBoundedExponentialBackoffRetry( + .retryPolicy(new CuratorRetryPolicy( ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL)), ObjectReader.getInt(conf .get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING)), @@ -224,22 +260,4 @@ public class CuratorUtils { builder, String zkStr, Map<String, Object> conf, ZookeeperAuthInfo auth) { setupBuilder(builder, zkStr, conf, auth); } - - public static CuratorFramework newCuratorStarted(Map<String, Object> conf, List<String> servers, - Object port, - String root, ZookeeperAuthInfo auth, List<ACL> defaultAcl) { - CuratorFramework ret = newCurator(conf, servers, port, root, auth, defaultAcl); - LOG.info("Starting Utils Curator..."); - ret.start(); - return ret; - } - - public static CuratorFramework newCuratorStarted(Map<String, Object> conf, List<String> servers, - Object port, - ZookeeperAuthInfo auth, List<ACL> defaultAcl) { - CuratorFramework ret = newCurator(conf, servers, port, auth, defaultAcl); - LOG.info("Starting Utils Curator (2)..."); - ret.start(); - return ret; - } } diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorZkClient.java b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorZkClient.java new file mode 100644 index 0000000000..31c737bcd9 --- /dev/null +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/CuratorZkClient.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper.curator; + +import java.util.List; +import java.util.Objects; +import org.apache.storm.cluster.ClusterStateListener; +import org.apache.storm.cluster.ConnectionState; +import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; +import org.apache.storm.shade.org.apache.zookeeper.CreateMode; +import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.shade.org.apache.zookeeper.data.Stat; +import org.apache.storm.zookeeper.ZkClient; + +/** + * Internal. The Curator-backed {@link ZkClient}. Every method is a one-to-one translation to the + * corresponding Curator builder chain, so behaviour is identical to calling Curator directly. + * + * <p>Not part of Storm's public API; may change or be removed without notice. + */ +public final class CuratorZkClient implements ZkClient { + private final CuratorFramework curator; + + CuratorZkClient(CuratorFramework curator) { + this.curator = Objects.requireNonNull(curator, "curator"); + } + + /** + * Wrap an already started {@link CuratorFramework}. Intended for tests that build their own + * Curator client. + * + * @param curator the started Curator client + * @return a {@link ZkClient} delegating to {@code curator} + */ + public static ZkClient wrap(CuratorFramework curator) { + return new CuratorZkClient(curator); + } + + /** + * Return the {@link CuratorFramework} backing a {@link ZkClient}. Only used by code that still + * needs Curator recipes directly (leader election). + * + * @param client a Curator-backed client + * @return the underlying Curator client + * @throws IllegalArgumentException if {@code client} is not Curator-backed + */ + public static CuratorFramework unwrap(ZkClient client) { + if (client instanceof CuratorZkClient) { + return ((CuratorZkClient) client).curator; + } + throw new IllegalArgumentException("ZkClient is not backed by Curator: " + + (client == null ? "null" : client.getClass().getName())); + } + + @Override + public String create(String path, byte[] data, CreateMode mode, List<ACL> acls, + boolean createParents) throws Exception { + if (createParents) { + return curator.create().creatingParentsIfNeeded().withMode(mode).withACL(acls) + .forPath(path, data); + } + return curator.create().withMode(mode).withACL(acls).forPath(path, data); + } + + @Override + public String createWithDefaultData(String path, CreateMode mode, List<ACL> acls, + boolean createParents) throws Exception { + if (createParents) { + return curator.create().creatingParentsIfNeeded().withMode(mode).withACL(acls) + .forPath(path); + } + return curator.create().withMode(mode).withACL(acls).forPath(path); + } + + @Override + public Stat exists(String path, boolean watch) throws Exception { + if (watch) { + return curator.checkExists().watched().forPath(path); + } + return curator.checkExists().forPath(path); + } + + @Override + public byte[] getData(String path, boolean watch, Stat stat) throws Exception { + if (stat != null) { + if (watch) { + return curator.getData().storingStatIn(stat).watched().forPath(path); + } + return curator.getData().storingStatIn(stat).forPath(path); + } + if (watch) { + return curator.getData().watched().forPath(path); + } + return curator.getData().forPath(path); + } + + @Override + public Stat setData(String path, byte[] data) throws Exception { + return curator.setData().forPath(path, data); + } + + @Override + public List<String> getChildren(String path, boolean watch) throws Exception { + if (watch) { + return curator.getChildren().watched().forPath(path); + } + return curator.getChildren().forPath(path); + } + + @Override + public void delete(String path, boolean recursive) throws Exception { + if (recursive) { + curator.delete().deletingChildrenIfNeeded().forPath(path); + } else { + curator.delete().forPath(path); + } + } + + @Override + public List<ACL> getAcl(String path) throws Exception { + return curator.getACL().forPath(path); + } + + @Override + public void setAcl(String path, List<ACL> acls) throws Exception { + curator.setACL().withACL(acls).forPath(path); + } + + @Override + public void sync(String path) throws Exception { + curator.sync().forPath(path); + } + + @Override + public void addConnectionStateListener(ClusterStateListener listener) { + Objects.requireNonNull(listener, "listener"); + curator.getConnectionStateListenable().addListener( + (unused, newState) -> listener.stateChanged(toStorm(newState))); + } + + @Override + public void close() { + curator.close(); + } + + /** + * Map a Curator connection state onto Storm's. An explicit switch so that a Curator upgrade + * adding a state fails loudly instead of being silently mis-mapped. + */ + static ConnectionState toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState state) { + switch (state) { + case CONNECTED: + return ConnectionState.CONNECTED; + case SUSPENDED: + return ConnectionState.SUSPENDED; + case RECONNECTED: + return ConnectionState.RECONNECTED; + case LOST: + return ConnectionState.LOST; + case READ_ONLY: + return ConnectionState.READ_ONLY; + default: + throw new IllegalStateException("Unknown Curator connection state " + state); + } + } +} diff --git a/storm-client/src/jvm/org/apache/storm/zookeeper/curator/package-info.java b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/package-info.java new file mode 100644 index 0000000000..a087dfa1db --- /dev/null +++ b/storm-client/src/jvm/org/apache/storm/zookeeper/curator/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Internal. Curator-backed implementation of {@link org.apache.storm.zookeeper.ZkClient}. + * + * <p>This package is not part of Storm's public API and may change or be removed without notice, + * in particular when Storm stops depending on Apache Curator. External code must use + * {@link org.apache.storm.zookeeper.ZkClient}, + * {@link org.apache.storm.zookeeper.ClientZookeeper#mkClient} and + * {@link org.apache.storm.cluster.ClusterStateListener} instead. + */ +package org.apache.storm.zookeeper.curator; diff --git a/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java b/storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorUtilsTest.java similarity index 98% rename from storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java rename to storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorUtilsTest.java index 08115d28dd..953aeffa4b 100644 --- a/storm-client/test/jvm/org/apache/storm/utils/CuratorUtilsTest.java +++ b/storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorUtilsTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.storm.utils; +package org.apache.storm.zookeeper.curator; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -38,6 +38,9 @@ import org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNetty; import org.apache.storm.shade.org.apache.zookeeper.ZooKeeper; import org.apache.storm.shade.org.apache.zookeeper.client.ZKClientConfig; import org.apache.storm.shade.org.apache.zookeeper.common.ClientX509Util; +import org.apache.storm.utils.ObjectReader; +import org.apache.storm.utils.Utils; +import org.apache.storm.utils.ZookeeperAuthInfo; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,8 +73,8 @@ public class CuratorUtilsTest { CuratorFramework curator = CuratorUtils.newCurator(config, Collections.singletonList("bogus_server"), 42, "", DaemonType.WORKER .getDefaultZkAcls(config)); - StormBoundedExponentialBackoffRetry policy = - (StormBoundedExponentialBackoffRetry) curator.getZookeeperClient().getRetryPolicy(); + CuratorRetryPolicy policy = + (CuratorRetryPolicy) curator.getZookeeperClient().getRetryPolicy(); assertEquals(policy.getBaseSleepTimeMs(), expectedInterval); assertEquals(policy.getN(), expectedRetries); assertEquals(policy.getSleepTimeMs(10, 0), expectedCeiling); diff --git a/storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorZkClientTest.java b/storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorZkClientTest.java new file mode 100644 index 0000000000..e1c57c75f5 --- /dev/null +++ b/storm-client/test/jvm/org/apache/storm/zookeeper/curator/CuratorZkClientTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper.curator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.apache.curator.test.TestingServer; +import org.apache.storm.Config; +import org.apache.storm.cluster.ConnectionState; +import org.apache.storm.shade.org.apache.zookeeper.CreateMode; +import org.apache.storm.shade.org.apache.zookeeper.KeeperException; +import org.apache.storm.shade.org.apache.zookeeper.Watcher; +import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; +import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.shade.org.apache.zookeeper.data.Stat; +import org.apache.storm.utils.Utils; +import org.apache.storm.zookeeper.ZkClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class CuratorZkClientTest { + /** + * A default ACL that differs from OPEN_ACL_UNSAFE (no ADMIN permission), so tests can tell the + * client default apart from Curator's fallback. + */ + private static final List<ACL> DEFAULT_ACL = Collections.singletonList( + new ACL(ZooDefs.Perms.ALL & ~ZooDefs.Perms.ADMIN, ZooDefs.Ids.ANYONE_ID_UNSAFE)); + + private TestingServer server; + private ZkClient client; + private final LinkedBlockingQueue<Watcher.Event.EventType> events = new LinkedBlockingQueue<>(); + + @BeforeEach + public void setUp() throws Exception { + server = new TestingServer(true); + Map<String, Object> conf = Utils.readDefaultConfig(); + conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 1); + client = CuratorUtils.newClient(conf, Collections.singletonList("127.0.0.1"), + server.getPort(), "", null, DEFAULT_ACL, + (state, type, path) -> { + // Session events (type None) are delivered too; only keep node events. + if (type != Watcher.Event.EventType.None) { + events.add(type); + } + }); + } + + @AfterEach + public void tearDown() throws Exception { + if (client != null) { + client.close(); + } + if (server != null) { + server.close(); + } + } + + @Test + public void createWithDefaultDataStoresLocalAddress() throws Exception { + client.createWithDefaultData("/dflt", CreateMode.PERSISTENT, null, false); + byte[] expected = InetAddress.getLocalHost().getHostAddress().getBytes(); + assertArrayEquals(expected, client.getData("/dflt", false, null)); + } + + @Test + public void createWithNullDataStoresNoData() throws Exception { + client.create("/nulldata", null, CreateMode.PERSISTENT, null, false); + byte[] data = client.getData("/nulldata", false, null); + assertTrue(data == null || data.length == 0, "expected no data"); + } + + @Test + public void createParentsUseClientDefaultAcl() throws Exception { + String path = client.create("/a/b/c", "x".getBytes(StandardCharsets.UTF_8), + CreateMode.PERSISTENT, ZooDefs.Ids.OPEN_ACL_UNSAFE, true); + assertEquals("/a/b/c", path); + assertEquals(DEFAULT_ACL, client.getAcl("/a")); + assertEquals(DEFAULT_ACL, client.getAcl("/a/b")); + assertEquals(ZooDefs.Ids.OPEN_ACL_UNSAFE, client.getAcl("/a/b/c")); + + client.create("/a/dflt", null, CreateMode.PERSISTENT, null, false); + assertEquals(DEFAULT_ACL, client.getAcl("/a/dflt")); + } + + @Test + public void aclRoundTrip() throws Exception { + client.create("/acl", null, CreateMode.PERSISTENT, ZooDefs.Ids.OPEN_ACL_UNSAFE, false); + client.setAcl("/acl", ZooDefs.Ids.READ_ACL_UNSAFE); + assertEquals(ZooDefs.Ids.READ_ACL_UNSAFE, client.getAcl("/acl")); + } + + @Test + public void dataStatAndVersions() throws Exception { + assertNull(client.exists("/v", false)); + client.create("/v", "1".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, null, + false); + Stat stat = client.setData("/v", "2".getBytes(StandardCharsets.UTF_8)); + assertEquals(1, stat.getVersion()); + Stat read = new Stat(); + assertArrayEquals("2".getBytes(StandardCharsets.UTF_8), client.getData("/v", false, read)); + assertEquals(1, read.getVersion()); + assertEquals(1, client.exists("/v", false).getVersion()); + } + + @Test + public void sequentialCreateReturnsActualPath() throws Exception { + String path = client.create("/seq-", null, CreateMode.PERSISTENT_SEQUENTIAL, null, false); + assertTrue(path.startsWith("/seq-") && path.length() > "/seq-".length(), path); + } + + @Test + public void watchesReachWatcherCallBack() throws Exception { + assertNull(client.exists("/w", true)); + client.create("/w", "a".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, null, + false); + assertEquals(Watcher.Event.EventType.NodeCreated, events.poll(30, TimeUnit.SECONDS)); + + client.getData("/w", true, null); + client.setData("/w", "b".getBytes(StandardCharsets.UTF_8)); + assertEquals(Watcher.Event.EventType.NodeDataChanged, events.poll(30, TimeUnit.SECONDS)); + + client.getChildren("/w", true); + client.create("/w/child", null, CreateMode.PERSISTENT, null, false); + assertEquals(Watcher.Event.EventType.NodeChildrenChanged, + events.poll(30, TimeUnit.SECONDS)); + } + + @Test + public void deleteRecursiveAndNonRecursive() throws Exception { + client.create("/d/e/f", null, CreateMode.PERSISTENT, null, true); + assertThrows(KeeperException.NotEmptyException.class, () -> client.delete("/d", false)); + client.delete("/d", true); + assertNull(client.exists("/d", false)); + } + + @Test + public void keeperExceptionsPropagateUnwrapped() throws Exception { + assertThrows(KeeperException.NoNodeException.class, + () -> client.getData("/missing", false, null)); + assertThrows(KeeperException.NoNodeException.class, + () -> client.setData("/missing", new byte[0])); + assertThrows(KeeperException.NoNodeException.class, + () -> client.delete("/missing", false)); + client.create("/dup", null, CreateMode.PERSISTENT, null, false); + assertThrows(KeeperException.NodeExistsException.class, + () -> client.create("/dup", null, CreateMode.PERSISTENT, null, false)); + } + + @Test + public void syncDoesNotFail() throws Exception { + client.create("/s", null, CreateMode.PERSISTENT, null, false); + client.sync("/s"); + } + + @Test + public void unwrapReturnsBackingCuratorAndRejectsOthers() { + assertNotNull(CuratorZkClient.unwrap(client)); + assertSame(CuratorZkClient.unwrap(client), + CuratorZkClient.unwrap(CuratorZkClient.wrap(CuratorZkClient.unwrap(client)))); + assertThrows(IllegalArgumentException.class, () -> CuratorZkClient.unwrap(null)); + } + + @Test + public void connectionStatesMapOneToOne() { + assertEquals(ConnectionState.CONNECTED, CuratorZkClient.toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.CONNECTED)); + assertEquals(ConnectionState.SUSPENDED, CuratorZkClient.toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.SUSPENDED)); + assertEquals(ConnectionState.RECONNECTED, CuratorZkClient.toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.RECONNECTED)); + assertEquals(ConnectionState.LOST, CuratorZkClient.toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.LOST)); + assertEquals(ConnectionState.READ_ONLY, CuratorZkClient.toStorm( + org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.READ_ONLY)); + // Every Curator state must be mapped (fails loudly if Curator adds one). + for (org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState s + : org.apache.storm.shade.org.apache.curator.framework.state.ConnectionState.values()) { + assertEquals(s.name(), CuratorZkClient.toStorm(s).name()); + } + } +} diff --git a/storm-core/test/jvm/org/apache/storm/trident/StateTest.java b/storm-core/test/jvm/org/apache/storm/trident/StateTest.java index f43a391b6c..c85f2a586d 100644 --- a/storm-core/test/jvm/org/apache/storm/trident/StateTest.java +++ b/storm-core/test/jvm/org/apache/storm/trident/StateTest.java @@ -19,9 +19,6 @@ package org.apache.storm.trident; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.api.CreateBuilder; -import org.apache.storm.shade.org.apache.curator.framework.api.ProtectACLCreateModeStatPathAndBytesable; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; @@ -36,6 +33,7 @@ import org.apache.storm.trident.testing.MemoryBackingMap; import org.apache.storm.trident.testing.MemoryMapState; import org.apache.storm.trident.topology.state.TestTransactionalState; import org.apache.storm.utils.Utils; +import org.apache.storm.zookeeper.ZkClient; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; @@ -141,16 +139,41 @@ public class StateTest { @Test public void testCreateNodeAcl() throws Exception { // Creates ZooKeeper nodes with the correct ACLs - CuratorFramework curator = Mockito.mock(CuratorFramework.class); - CreateBuilder builder0 = Mockito.mock(CreateBuilder.class); - ProtectACLCreateModeStatPathAndBytesable builder1 = Mockito.mock(ProtectACLCreateModeStatPathAndBytesable.class); + ZkClient zk = Mockito.mock(ZkClient.class); List<ACL> expectedAcls = ZooDefs.Ids.CREATOR_ALL_ACL; - Mockito.when(curator.create()).thenReturn(builder0); - Mockito.when(builder0.creatingParentsIfNeeded()).thenReturn(builder1); - Mockito.when(builder1.withMode(ArgumentMatchers.isA(CreateMode.class))).thenReturn(builder1); - Mockito.when(builder1.withACL(Mockito.anyList())).thenReturn(builder1); - TestTransactionalState.createNode(curator, "", new byte[0], expectedAcls, null); - Mockito.verify(builder1).withACL(expectedAcls); + TestTransactionalState.createNode(zk, "", new byte[0], expectedAcls, null); + Mockito.verify(zk).create(ArgumentMatchers.eq(""), ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(CreateMode.PERSISTENT), ArgumentMatchers.eq(expectedAcls), + ArgumentMatchers.eq(true)); + Mockito.verifyNoMoreInteractions(zk); + } + + @Test + public void testCreateNodeAclIgnoresModeWhenAclsGiven() throws Exception { + // As with the former Curator builder chain, the mode is only honoured without explicit ACLs + ZkClient zk = Mockito.mock(ZkClient.class); + List<ACL> expectedAcls = ZooDefs.Ids.CREATOR_ALL_ACL; + TestTransactionalState.createNode(zk, "/n", new byte[]{1}, expectedAcls, CreateMode.EPHEMERAL); + Mockito.verify(zk).create(ArgumentMatchers.eq("/n"), ArgumentMatchers.any(byte[].class), + ArgumentMatchers.eq(CreateMode.PERSISTENT), ArgumentMatchers.eq(expectedAcls), + ArgumentMatchers.eq(true)); + } + + @Test + public void testCreateNodeNullDataUsesDefaultData() throws Exception { + ZkClient zk = Mockito.mock(ZkClient.class); + TestTransactionalState.createNode(zk, "/root", null, null, null); + Mockito.verify(zk).createWithDefaultData("/root", CreateMode.PERSISTENT, null, true); + Mockito.verifyNoMoreInteractions(zk); + } + + @Test + public void testCreateNodeNullAclsKeepsMode() throws Exception { + ZkClient zk = Mockito.mock(ZkClient.class); + byte[] data = new byte[]{1, 2}; + TestTransactionalState.createNode(zk, "/eph", data, null, CreateMode.EPHEMERAL); + Mockito.verify(zk).create("/eph", data, CreateMode.EPHEMERAL, null, true); + Mockito.verifyNoMoreInteractions(zk); } @Test diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java index f42b0c7e03..8b0fb563c9 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/BlobStoreUtils.java @@ -29,13 +29,12 @@ import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.generated.ReadableBlobMeta; import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.security.auth.NimbusPrincipal; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.thrift.transport.TTransportException; -import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.NimbusClient; import org.apache.storm.utils.WrappedKeyNotFoundException; -import org.apache.storm.utils.ZookeeperAuthInfo; +import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,16 +48,12 @@ public class BlobStoreUtils { } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") - public static CuratorFramework createZKClient(Map<String, Object> conf, DaemonType type) { + public static ZkClient createZKClient(Map<String, Object> conf, DaemonType type) { @SuppressWarnings("unchecked") List<String> zkServers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); Object port = conf.get(Config.STORM_ZOOKEEPER_PORT); - ZookeeperAuthInfo zkAuthInfo = new ZookeeperAuthInfo(conf); - CuratorFramework zkClient = CuratorUtils.newCurator(conf, zkServers, port, - (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), zkAuthInfo, - type.getDefaultZkAcls(conf)); - zkClient.start(); - return zkClient; + return ClientZookeeper.mkClient(conf, zkServers, port, (String) conf.get(Config.STORM_ZOOKEEPER_ROOT), null, conf, + type); } public static Subject getNimbusSubject() { @@ -77,10 +72,10 @@ public class BlobStoreUtils { } // Check for latest sequence number of a key inside zookeeper and return nimbodes containing the latest sequence number - public static Set<NimbusInfo> getNimbodesWithLatestSequenceNumberOfBlob(CuratorFramework zkClient, String key) throws Exception { + public static Set<NimbusInfo> getNimbodesWithLatestSequenceNumberOfBlob(ZkClient zkClient, String key) throws Exception { List<String> stateInfoList; try { - stateInfoList = zkClient.getChildren().forPath("/blobstore/" + key); + stateInfoList = zkClient.getChildren("/blobstore/" + key, false); } catch (KeeperException.NoNodeException e) { // there's a race condition with a delete: blobstore // this should be thrown to the caller to indicate that the key is invalid now @@ -243,7 +238,7 @@ public class BlobStoreUtils { cb.createStateInZookeeper(key); } - public static void updateKeyForBlobStore(Map<String, Object> conf, BlobStore blobStore, CuratorFramework zkClient, String key, + public static void updateKeyForBlobStore(Map<String, Object> conf, BlobStore blobStore, ZkClient zkClient, String key, NimbusInfo nimbusDetails) { try { // Most of clojure tests currently try to access the blobs using getBlob. Since, updateKeyForBlobStore @@ -257,10 +252,10 @@ public class BlobStoreUtils { } boolean isListContainsCurrentNimbusInfo = false; List<String> stateInfo; - if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + "/" + key) == null) { + if (zkClient.exists(BLOBSTORE_SUBTREE + "/" + key, false) == null) { return; } - stateInfo = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + "/" + key); + stateInfo = zkClient.getChildren(BLOBSTORE_SUBTREE + "/" + key, false); if (stateInfo == null || stateInfo.isEmpty()) { return; } diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java index ae0f998603..ea0117d347 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/KeySequenceNumber.java @@ -17,11 +17,11 @@ import java.util.List; import java.util.TreeSet; import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.nimbus.NimbusInfo; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; import org.apache.storm.utils.WrappedKeyNotFoundException; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,7 +123,7 @@ public class KeySequenceNumber { this.nimbusInfo = nimbusInfo; } - public synchronized int getKeySequenceNumber(CuratorFramework zkClient) throws KeyNotFoundException { + public synchronized int getKeySequenceNumber(ZkClient zkClient) throws KeyNotFoundException { return getKeySequenceNumber(zkClient, true); } @@ -137,17 +137,17 @@ public class KeySequenceNumber { * @return the sequence number * @throws KeyNotFoundException if the key is not in zookeeper and may not be created, or it is deleted meanwhile */ - public synchronized int getKeySequenceNumber(CuratorFramework zkClient, boolean mayCreateKey) throws KeyNotFoundException { + public synchronized int getKeySequenceNumber(ZkClient zkClient, boolean mayCreateKey) throws KeyNotFoundException { TreeSet<Integer> sequenceNumbers = new TreeSet<Integer>(); try { // Key has not been created yet and it is the first time it is being created - if (zkClient.checkExists().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key) == null) { + if (zkClient.exists(BlobStoreUtils.getBlobStoreSubtree() + "/" + key, false) == null) { if (!mayCreateKey) { throw new KeeperException.NoNodeException(BlobStoreUtils.getBlobStoreSubtree() + "/" + key); } - zkClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT) - .withACL(ZooDefs.Ids.OPEN_ACL_UNSAFE).forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key); - zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, + zkClient.createWithDefaultData(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, CreateMode.PERSISTENT, + ZooDefs.Ids.OPEN_ACL_UNSAFE, true); + zkClient.setData(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, ByteBuffer.allocate(INT_CAPACITY).putInt(INITIAL_SEQUENCE_NUMBER).array()); return INITIAL_SEQUENCE_NUMBER; } @@ -155,7 +155,7 @@ public class KeySequenceNumber { // When all nimbodes go down and one or few of them come up // Unfortunately there might not be an exact way to know which one contains the most updated blob, // if all go down which is unlikely. Hence there might be a need to update the blob if all go down. - List<String> stateInfoList = zkClient.getChildren().forPath(BlobStoreUtils.getBlobStoreSubtree() + "/" + key); + List<String> stateInfoList = zkClient.getChildren(BlobStoreUtils.getBlobStoreSubtree() + "/" + key, false); LOG.debug("stateInfoList-size {} stateInfoList-data {}", stateInfoList.size(), stateInfoList); if (stateInfoList.isEmpty()) { return getMaxSequenceNumber(zkClient); @@ -229,13 +229,12 @@ public class KeySequenceNumber { return containsNimbusHost; } - private void incrementMaxSequenceNumber(CuratorFramework zkClient, int count) throws Exception { - zkClient.setData().forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, - ByteBuffer.allocate(INT_CAPACITY).putInt(count + 1).array()); + private void incrementMaxSequenceNumber(ZkClient zkClient, int count) throws Exception { + zkClient.setData(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, + ByteBuffer.allocate(INT_CAPACITY).putInt(count + 1).array()); } - private int getMaxSequenceNumber(CuratorFramework zkClient) throws Exception { - return ByteBuffer.wrap(zkClient.getData() - .forPath(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key)).getInt(); + private int getMaxSequenceNumber(ZkClient zkClient) throws Exception { + return ByteBuffer.wrap(zkClient.getData(BLOBSTORE_MAX_KEY_SEQUENCE_SUBTREE + "/" + key, false, null)).getInt(); } } diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java index a5d5c5bcf8..717087d399 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java @@ -48,13 +48,13 @@ import org.apache.storm.generated.SettableBlobMeta; import org.apache.storm.nimbus.ILeaderElector; import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.shade.com.google.common.annotations.VisibleForTesting; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.utils.ConfigUtils; import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Utils; import org.apache.storm.utils.WrappedKeyAlreadyExistsException; import org.apache.storm.utils.WrappedKeyNotFoundException; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,7 +87,7 @@ public class LocalFsBlobStore extends BlobStore { private NimbusInfo nimbusInfo; private FileBlobStoreImpl fbs; private Map<String, Object> conf; - private CuratorFramework zkClient; + private ZkClient zkClient; private IStormClusterState stormClusterState; private Timer timer; private ILeaderElector leaderElector; @@ -438,11 +438,11 @@ public class LocalFsBlobStore extends BlobStore { validateKey(key); SettableBlobMeta meta = getStoredBlobMeta(key); aclHandler.hasPermissions(meta.get_acl(), READ, who, key); - if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + key) == null) { + if (zkClient.exists(BLOBSTORE_SUBTREE + key, false) == null) { return 0; } try { - replicationCount = zkClient.getChildren().forPath(BLOBSTORE_SUBTREE + key).size(); + replicationCount = zkClient.getChildren(BLOBSTORE_SUBTREE + key, false).size(); } catch (KeeperException.NoNodeException e) { //Race with delete //If it is not here the replication is 0 @@ -462,7 +462,7 @@ public class LocalFsBlobStore extends BlobStore { try { List<String> keyList = BlobStoreUtils.getKeyListFromBlobStore(this); if (!keyList.contains(key)) { - if (zkClient.checkExists().forPath(BLOBSTORE_SUBTREE + key) != null) { + if (zkClient.exists(BLOBSTORE_SUBTREE + key, false) != null) { Set<NimbusInfo> nimbusSet = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, key); nimbusSet.remove(this.nimbusInfo); if (BlobStoreUtils.downloadMissingBlob(conf, this, key, nimbusSet)) { diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java index 77ef8c9be9..3d21bb85ef 100644 --- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java +++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizer.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.Set; import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.nimbus.NimbusInfo; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory; */ public class LocalFsBlobStoreSynchronizer { private static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStoreSynchronizer.class); - private CuratorFramework zkClient; + private ZkClient zkClient; private Map<String, Object> conf; private BlobStore blobStore; private Set<String> blobStoreKeySet = new HashSet<String>(); @@ -53,7 +53,7 @@ public class LocalFsBlobStoreSynchronizer { this.nimbusInfo = nimbusInfo; } - public void setZkClient(CuratorFramework zkClient) { + public void setZkClient(ZkClient zkClient) { this.zkClient = zkClient; } diff --git a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java index eab1fe49e4..dbc26c556a 100644 --- a/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java +++ b/storm-server/src/main/java/org/apache/storm/daemon/nimbus/Nimbus.java @@ -205,7 +205,6 @@ import org.apache.storm.shade.com.google.common.base.Strings; import org.apache.storm.shade.com.google.common.collect.ImmutableMap; import org.apache.storm.shade.com.google.common.collect.MapDifference; import org.apache.storm.shade.com.google.common.collect.Maps; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.stats.ClientStatsUtil; @@ -234,6 +233,7 @@ import org.apache.storm.utils.WrappedNotAliveException; import org.apache.storm.validation.ConfigValidation; import org.apache.storm.zookeeper.AclEnforcement; import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; import org.apache.storm.zookeeper.Zookeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -476,8 +476,8 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { private final ClusterSummaryMetricSet clusterMetricSet; private MetricStore metricsStore; private IAuthorizer authorizationHandler; - //Cached CuratorFramework, mainly used for BlobStore. - private final CuratorFramework zkClient; + //Cached zookeeper client, mainly used for BlobStore. + private final ZkClient zkClient; //Cached topology -> executor ids, used for deciding timeout workers of heartbeatsCache. private AtomicReference<Map<String, Set<List<Integer>>>> idToExecutors; //May be null if worker tokens are not supported by the thrift transport. @@ -775,17 +775,17 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { } public static int getVersionForKey(String key, NimbusInfo nimbusInfo, - CuratorFramework zkClient) throws KeyNotFoundException { + ZkClient zkClient) throws KeyNotFoundException { return getVersionForKey(key, nimbusInfo, zkClient, true); } /** * Get the version to register the copy of a blob held by a nimbus under. * - * @see KeySequenceNumber#getKeySequenceNumber(CuratorFramework, boolean) + * @see KeySequenceNumber#getKeySequenceNumber(ZkClient, boolean) */ public static int getVersionForKey(String key, NimbusInfo nimbusInfo, - CuratorFramework zkClient, boolean mayCreateKey) throws KeyNotFoundException { + ZkClient zkClient, boolean mayCreateKey) throws KeyNotFoundException { KeySequenceNumber kseq = new KeySequenceNumber(key, nimbusInfo); return kseq.getKeySequenceNumber(zkClient, mayCreateKey); } @@ -1718,11 +1718,11 @@ public class Nimbus implements Iface, Shutdownable, DaemonCommon { } @SuppressWarnings("checkstyle:AbbreviationAsWordInName") - private static CuratorFramework makeZKClient(Map<String, Object> conf) { + private static ZkClient makeZKClient(Map<String, Object> conf) { List<String> servers = (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS); Object port = conf.get(Config.STORM_ZOOKEEPER_PORT); String root = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT); - CuratorFramework ret = null; + ZkClient ret = null; if (servers != null && port != null) { ret = ClientZookeeper.mkClient(conf, servers, port, root, new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS); } diff --git a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java index 5d36c8168d..5af41fe5ca 100644 --- a/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java +++ b/storm-server/src/main/java/org/apache/storm/nimbus/LeaderListenerCallback.java @@ -37,13 +37,12 @@ import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.security.auth.ReqContext; import org.apache.storm.shade.com.google.common.base.Joiner; import org.apache.storm.shade.com.google.common.collect.Sets; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.utils.ObjectReader; import org.apache.storm.utils.Utils; import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,7 +61,7 @@ public class LeaderListenerCallback { private final BlobStore blobStore; private final TopoCache tc; private final IStormClusterState clusterState; - private final CuratorFramework zk; + private final ZkClient zk; private final ILeaderElector leaderElector; private final Map conf; private final List<ACL> acls; @@ -72,14 +71,14 @@ public class LeaderListenerCallback { * Creates a {@link LeaderListenerCallback}. * * @param conf config - * @param zk zookeeper CuratorFramework client + * @param zk zookeeper client * @param blobStore BlobStore * @param leaderElector Leader elector * @param tc TopoCache * @param clusterState IStormClusterState * @param acls zookeeper acls */ - public LeaderListenerCallback(Map conf, CuratorFramework zk, BlobStore blobStore, ILeaderElector leaderElector, + public LeaderListenerCallback(Map conf, ZkClient zk, BlobStore blobStore, ILeaderElector leaderElector, TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry) { this.blobStore = blobStore; this.tc = tc; diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java index 1ecdfe36e5..caae64cb0b 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/AclEnforcement.java @@ -30,7 +30,7 @@ import org.apache.storm.generated.KeyNotFoundException; import org.apache.storm.generated.WorkerTokenServiceType; import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.security.auth.NimbusPrincipal; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; +import org.apache.storm.shade.org.apache.zookeeper.CreateMode; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; @@ -79,9 +79,9 @@ public class AclEnforcement { int port = ObjectReader.getInt(conf.get(Config.STORM_ZOOKEEPER_PORT)); String stormRoot = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT); - try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, "", - new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { - if (zk.checkExists().forPath(stormRoot) != null) { + try (ZkClient zk = ClientZookeeper.mkClient(conf, zkServers, port, "", + new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { + if (zk.exists(stormRoot, false) != null) { //First off we want to verify that ROOT is good verifyAclStrict(zk, superAcl, stormRoot, fixUp); } else { @@ -91,21 +91,21 @@ public class AclEnforcement { } // Now that the root is fine we can start to look at the other paths under it. - try (CuratorFramework zk = ClientZookeeper.mkClient(conf, zkServers, port, stormRoot, - new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { + try (ZkClient zk = ClientZookeeper.mkClient(conf, zkServers, port, stormRoot, + new DefaultWatcherCallBack(), conf, DaemonType.NIMBUS)) { //Next verify that the blob store is correct before we start it up. - if (zk.checkExists().forPath(ClusterUtils.BLOBSTORE_SUBTREE) != null) { + if (zk.exists(ClusterUtils.BLOBSTORE_SUBTREE, false) != null) { verifyAclStrictRecursive(zk, superAcl, ClusterUtils.BLOBSTORE_SUBTREE, fixUp); } - if (zk.checkExists().forPath(ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE) != null) { + if (zk.exists(ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE, false) != null) { verifyAclStrict(zk, superAcl, ClusterUtils.BLOBSTORE_MAX_KEY_SEQUENCE_NUMBER_SUBTREE, fixUp); } //The blobstore is good, now lets get the list of all topo Ids Set<String> topoIds = new HashSet<>(); - if (zk.checkExists().forPath(ClusterUtils.STORMS_SUBTREE) != null) { - topoIds.addAll(zk.getChildren().forPath(ClusterUtils.STORMS_SUBTREE)); + if (zk.exists(ClusterUtils.STORMS_SUBTREE, false) != null) { + topoIds.addAll(zk.getChildren(ClusterUtils.STORMS_SUBTREE, false)); } Map<String, Id> topoToZkCreds = new HashMap<>(); @@ -143,14 +143,15 @@ public class AclEnforcement { //There is a race on backpressure too... verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.BACKPRESSURE_SUBTREE, topoToZkCreds, fixUp); - if (zk.checkExists().forPath(ClusterUtils.ERRORS_SUBTREE) != null) { + if (zk.exists(ClusterUtils.ERRORS_SUBTREE, false) != null) { //errors is a bit special because in older versions of storm the worker created the parent directories lazily // because of this it means we need to auto create at least the topo-id directory for all running topos. for (String topoId : topoToZkCreds.keySet()) { String path = ClusterUtils.errorStormRoot(topoId); - if (zk.checkExists().forPath(path) == null) { + if (zk.exists(path, false) == null) { LOG.warn("Creating missing errors location {}", path); - zk.create().withACL(getTopoReadWrite(path, topoId, topoToZkCreds, superUserAcl, fixUp)).forPath(path); + zk.createWithDefaultData(path, CreateMode.PERSISTENT, + getTopoReadWrite(path, topoId, topoToZkCreds, superUserAcl, fixUp), false); } } } @@ -158,25 +159,25 @@ public class AclEnforcement { // for some odd reason they are leaked. verifyParentWithReadWriteTopoChildrenDeleteDead(zk, superUserAcl, ClusterUtils.ERRORS_SUBTREE, topoToZkCreds, fixUp); - if (zk.checkExists().forPath(ClusterUtils.SECRET_KEYS_SUBTREE) != null) { + if (zk.exists(ClusterUtils.SECRET_KEYS_SUBTREE, false) != null) { verifyAclStrict(zk, superAcl, ClusterUtils.SECRET_KEYS_SUBTREE, fixUp); verifyAclStrictRecursive(zk, superAcl, ClusterUtils.secretKeysPath(WorkerTokenServiceType.NIMBUS), fixUp); verifyAclStrictRecursive(zk, drpcFullAcl, ClusterUtils.secretKeysPath(WorkerTokenServiceType.DRPC), fixUp); } - if (zk.checkExists().forPath(ClusterUtils.NIMBUSES_SUBTREE) != null) { + if (zk.exists(ClusterUtils.NIMBUSES_SUBTREE, false) != null) { verifyAclStrictRecursive(zk, superAcl, ClusterUtils.NIMBUSES_SUBTREE, fixUp); } - if (zk.checkExists().forPath("/leader-lock") != null) { + if (zk.exists("/leader-lock", false) != null) { verifyAclStrictRecursive(zk, superAcl, "/leader-lock", fixUp); } - if (zk.checkExists().forPath(ClusterUtils.PROFILERCONFIG_SUBTREE) != null) { + if (zk.exists(ClusterUtils.PROFILERCONFIG_SUBTREE, false) != null) { verifyAclStrictRecursive(zk, superAcl, ClusterUtils.PROFILERCONFIG_SUBTREE, fixUp); } - if (zk.checkExists().forPath(ClusterUtils.SUPERVISORS_SUBTREE) != null) { + if (zk.exists(ClusterUtils.SUPERVISORS_SUBTREE, false) != null) { verifyAclStrictRecursive(zk, superAcl, ClusterUtils.SUPERVISORS_SUBTREE, fixUp); } @@ -204,12 +205,12 @@ public class AclEnforcement { return getTopoAcl(path, topoId, topoToZkCreds, superAcl, fixUp, ZooDefs.Perms.ALL); } - private static void verifyParentWithTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithTopoChildrenDeleteDead(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp, int perms) throws Exception { - if (zk.checkExists().forPath(path) != null) { + if (zk.exists(path, false) != null) { verifyAclStrict(zk, Arrays.asList(superUserAcl), path, fixUp); Set<String> possiblyBadIds = new HashSet<>(); - for (String topoId : zk.getChildren().forPath(path)) { + for (String topoId : zk.getChildren(path, false)) { String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; if (!topoToZkCreds.containsKey(topoId)) { //Save it to try again later... @@ -222,31 +223,31 @@ public class AclEnforcement { if (!possiblyBadIds.isEmpty()) { //Lets reread the children in STORMS as the source of truth and see if a new one was created in the background - possiblyBadIds.removeAll(zk.getChildren().forPath(ClusterUtils.STORMS_SUBTREE)); + possiblyBadIds.removeAll(zk.getChildren(ClusterUtils.STORMS_SUBTREE, false)); for (String topoId : possiblyBadIds) { //Now we know for sure that this is a bad id String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; - zk.delete().deletingChildrenIfNeeded().forPath(childPath); + zk.delete(childPath, true); } } } } - private static void verifyParentWithReadOnlyTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadOnlyTopoChildrenDeleteDead(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception { verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.READ); } - private static void verifyParentWithReadWriteTopoChildrenDeleteDead(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadWriteTopoChildrenDeleteDead(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception { verifyParentWithTopoChildrenDeleteDead(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.ALL); } - private static void verifyParentWithTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithTopoChildren(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp, int perms) throws Exception { - if (zk.checkExists().forPath(path) != null) { + if (zk.exists(path, false) != null) { verifyAclStrict(zk, Arrays.asList(superUserAcl), path, fixUp); - for (String topoId : zk.getChildren().forPath(path)) { + for (String topoId : zk.getChildren(path, false)) { String childPath = path + ClusterUtils.ZK_SEPERATOR + topoId; List<ACL> rwAcl = getTopoAcl(path, topoId, topoToZkCreds, superUserAcl, fixUp, perms); verifyAclStrictRecursive(zk, rwAcl, childPath, fixUp); @@ -254,31 +255,31 @@ public class AclEnforcement { } } - private static void verifyParentWithReadOnlyTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadOnlyTopoChildren(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception { verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.READ); } - private static void verifyParentWithReadWriteTopoChildren(CuratorFramework zk, ACL superUserAcl, String path, + private static void verifyParentWithReadWriteTopoChildren(ZkClient zk, ACL superUserAcl, String path, Map<String, Id> topoToZkCreds, boolean fixUp) throws Exception { verifyParentWithTopoChildren(zk, superUserAcl, path, topoToZkCreds, fixUp, ZooDefs.Perms.ALL); } - private static void verifyAclStrictRecursive(CuratorFramework zk, List<ACL> strictAcl, String path, boolean fixUp) throws Exception { + private static void verifyAclStrictRecursive(ZkClient zk, List<ACL> strictAcl, String path, boolean fixUp) throws Exception { verifyAclStrict(zk, strictAcl, path, fixUp); - for (String child : zk.getChildren().forPath(path)) { + for (String child : zk.getChildren(path, false)) { String newPath = path + ClusterUtils.ZK_SEPERATOR + child; verifyAclStrictRecursive(zk, strictAcl, newPath, fixUp); } } - private static void verifyAclStrict(CuratorFramework zk, List<ACL> strictAcl, String path, boolean fixUp) throws Exception { + private static void verifyAclStrict(ZkClient zk, List<ACL> strictAcl, String path, boolean fixUp) throws Exception { try { - List<ACL> foundAcl = zk.getACL().forPath(path); + List<ACL> foundAcl = zk.getAcl(path); if (!equivalent(foundAcl, strictAcl)) { if (fixUp) { LOG.warn("{} expected to have ACL {}, but has {}. Fixing...", path, strictAcl, foundAcl); - zk.setACL().withACL(strictAcl).forPath(path); + zk.setAcl(path, strictAcl); } else { throw new IllegalStateException(path + " did not have the correct ACL found " + foundAcl + " expected " + strictAcl); } diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java index 7e887fbb43..6ab88b0a91 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderElectorImp.java @@ -24,11 +24,20 @@ import org.apache.storm.nimbus.NimbusInfo; import org.apache.storm.shade.com.google.common.annotations.VisibleForTesting; import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch; +import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatchListener; import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.Participant; import org.apache.storm.utils.Utils; +import org.apache.storm.zookeeper.curator.CuratorZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * ZooKeeper based {@link ILeaderElector}. + * + * <p>Leader election is still implemented with Curator's {@code LeaderLatch} recipe, which is why + * the {@link ZkClient} passed in must be Curator-backed (as all clients created by + * {@link ClientZookeeper#mkClient} are). This is an implementation detail and not part of the API. + */ public class LeaderElectorImp implements ILeaderElector { private static final Logger LOG = LoggerFactory.getLogger(LeaderElectorImp.class); private final CuratorFramework zk; @@ -38,10 +47,10 @@ public class LeaderElectorImp implements ILeaderElector { private final LeaderListenerCallbackFactory leaderListenerCallbackFactory; private final StormTimer timer; - public LeaderElectorImp(CuratorFramework zk, String id, LeaderListenerCallbackFactory leaderListenerCallbackFactory) { - this.zk = zk; + public LeaderElectorImp(ZkClient zk, String id, LeaderListenerCallbackFactory leaderListenerCallbackFactory) { + this.zk = CuratorZkClient.unwrap(zk); this.id = id; - this.leaderLatch = new AtomicReference<>(new LeaderLatch(zk, leaderLockPath, id)); + this.leaderLatch = new AtomicReference<>(new LeaderLatch(this.zk, leaderLockPath, id)); this.leaderListenerCallbackFactory = leaderListenerCallbackFactory; this.timer = new StormTimer("leader-elector-timer", Utils.createDefaultUncaughtExceptionHandler()); } @@ -56,14 +65,14 @@ public class LeaderElectorImp implements ILeaderElector { // if this latch is closed, we need to create new instance. if (LeaderLatch.State.CLOSED.equals(leaderLatch.get().getState())) { LeaderLatch latch = new LeaderLatch(zk, leaderLockPath, id); - latch.addListener(leaderListenerCallbackFactory.create(this)); + latch.addListener(toLatchListener(leaderListenerCallbackFactory.create(this))); latch.start(); leaderLatch.set(latch); LOG.info("LeaderLatch was in closed state. Reset the leaderLatch, and queued for leader lock."); } // If the latch is not started yet, start it if (LeaderLatch.State.LATENT.equals(leaderLatch.get().getState())) { - leaderLatch.get().addListener(leaderListenerCallbackFactory.create(this)); + leaderLatch.get().addListener(toLatchListener(leaderListenerCallbackFactory.create(this))); leaderLatch.get().start(); LOG.info("Queued up for leader lock."); } else { @@ -106,7 +115,8 @@ public class LeaderElectorImp implements ILeaderElector { @Override public NimbusInfo getLeader() { try { - return Zookeeper.toNimbusInfo(leaderLatch.get().getLeader()); + Participant leader = leaderLatch.get().getLeader(); + return Zookeeper.toNimbusInfo(leader.getId(), leader.isLeader()); } catch (Exception e) { throw Utils.wrapInRuntime(e); } @@ -117,7 +127,7 @@ public class LeaderElectorImp implements ILeaderElector { List<NimbusInfo> nimbusInfos = new ArrayList<>(); Collection<Participant> participants = leaderLatch.get().getParticipants(); for (Participant participant : participants) { - nimbusInfos.add(Zookeeper.toNimbusInfo(participant)); + nimbusInfos.add(Zookeeper.toNimbusInfo(participant.getId(), participant.isLeader())); } return nimbusInfos; } @@ -127,4 +137,18 @@ public class LeaderElectorImp implements ILeaderElector { timer.close(); removeFromLeaderLockQueue(); } + + private static LeaderLatchListener toLatchListener(LeaderListener listener) { + return new LeaderLatchListener() { + @Override + public void isLeader() { + listener.isLeader(); + } + + @Override + public void notLeader() { + listener.notLeader(); + } + }; + } } diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListener.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListener.java new file mode 100644 index 0000000000..5f4196604e --- /dev/null +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListener.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper; + +/** + * Notified by a {@link LeaderElectorImp} when this process gains or loses leadership. + * + * <p>Callbacks are invoked on the leader elector's notification thread, one at a time. + */ +public interface LeaderListener { + + /** + * Called when this process has become the leader. + */ + void isLeader(); + + /** + * Called when this process is no longer the leader. + */ + void notLeader(); +} diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java index 5582c06035..2aa739f981 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/LeaderListenerCallbackFactory.java @@ -28,8 +28,6 @@ import org.apache.storm.daemon.nimbus.TopoCache; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.nimbus.ILeaderElector; import org.apache.storm.nimbus.LeaderListenerCallback; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatchListener; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +37,7 @@ public class LeaderListenerCallbackFactory { private static final Logger LOG = LoggerFactory.getLogger(LeaderListenerCallbackFactory.class); private final Map<String, Object> conf; - private final CuratorFramework zk; + private final ZkClient zk; private final BlobStore blobStore; private final TopoCache tc; private final IStormClusterState clusterState; @@ -47,7 +45,7 @@ public class LeaderListenerCallbackFactory { private final StormMetricsRegistry metricsRegistry; private final Object submitLock; - public LeaderListenerCallbackFactory(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore, TopoCache tc, + public LeaderListenerCallbackFactory(Map<String, Object> conf, ZkClient zk, BlobStore blobStore, TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry, Object submitLock) { this.conf = conf; this.zk = zk; @@ -59,11 +57,11 @@ public class LeaderListenerCallbackFactory { this.submitLock = submitLock; } - public LeaderLatchListener create(ILeaderElector elector) throws UnknownHostException { + public LeaderListener create(ILeaderElector elector) throws UnknownHostException { final LeaderListenerCallback callback = new LeaderListenerCallback(conf, zk, blobStore, elector, tc, clusterState, acls, metricsRegistry); final String hostName = InetAddress.getLocalHost().getCanonicalHostName(); - return new LeaderLatchListener() { + return new LeaderListener() { @Override public void isLeader() { diff --git a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java index a5a8ecac8c..7e1939b3d6 100644 --- a/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java +++ b/storm-server/src/main/java/org/apache/storm/zookeeper/Zookeeper.java @@ -30,8 +30,6 @@ import org.apache.storm.daemon.nimbus.TopoCache; import org.apache.storm.metric.StormMetricsRegistry; import org.apache.storm.nimbus.ILeaderElector; import org.apache.storm.nimbus.NimbusInfo; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.recipes.leader.Participant; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.shade.org.apache.zookeeper.server.NIOServerCnxnFactory; import org.apache.storm.shade.org.apache.zookeeper.server.ZooKeeperServer; @@ -95,13 +93,22 @@ public class Zookeeper { handle.shutdown(); } - public static NimbusInfo toNimbusInfo(Participant participant) { - String id = participant.getId(); + /** + * Convert a leader election participant into a {@link NimbusInfo}. + * + * @param participantId the participant id, which is the nimbus {@code host:port} string + * @param isLeader whether the participant is the current leader + * @return the parsed nimbus info + * @throws RuntimeException if {@code participantId} is blank, which is the case when there is + * no leader + */ + public static NimbusInfo toNimbusInfo(String participantId, boolean isLeader) { + String id = participantId; if (StringUtils.isBlank(id)) { throw new RuntimeException("No nimbus leader participant host found, have you started your nimbus hosts?"); } NimbusInfo nimbusInfo = NimbusInfo.parse(id); - nimbusInfo.setLeader(participant.isLeader()); + nimbusInfo.setLeader(isLeader); return nimbusInfo; } @@ -116,13 +123,13 @@ public class Zookeeper { * @param acls ACLs * @return Instance of {@link ILeaderElector} */ - public static ILeaderElector zkLeaderElector(Map<String, Object> conf, CuratorFramework zkClient, BlobStore blobStore, + public static ILeaderElector zkLeaderElector(Map<String, Object> conf, ZkClient zkClient, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry, Object submitLock) { return instance.zkLeaderElectorImpl(conf, zkClient, blobStore, tc, clusterState, acls, metricsRegistry, submitLock); } - protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, CuratorFramework zk, BlobStore blobStore, + protected ILeaderElector zkLeaderElectorImpl(Map<String, Object> conf, ZkClient zk, BlobStore blobStore, final TopoCache tc, IStormClusterState clusterState, List<ACL> acls, StormMetricsRegistry metricsRegistry, Object submitLock) { String id = NimbusInfo.fromConf(conf).toHostPortString(); diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java index 8b4e8a08ba..4968b00cef 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/KeySequenceNumberTest.java @@ -28,6 +28,7 @@ import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.storm.shade.org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.storm.testing.InProcessZookeeper; +import org.apache.storm.zookeeper.curator.CuratorZkClient; import org.junit.jupiter.api.Test; class KeySequenceNumberTest { @@ -82,7 +83,7 @@ class KeySequenceNumberTest { * Do what IStormClusterState#setupBlob does with the version KeySequenceNumber hands out. */ private static int register(CuratorFramework zkClient, NimbusInfo nimbus, boolean mayCreateKey) throws Exception { - int version = new KeySequenceNumber(KEY, nimbus).getKeySequenceNumber(zkClient, mayCreateKey); + int version = new KeySequenceNumber(KEY, nimbus).getKeySequenceNumber(CuratorZkClient.wrap(zkClient), mayCreateKey); if (zkClient.checkExists().forPath(KEY_PATH) != null) { for (String child : zkClient.getChildren().forPath(KEY_PATH)) { if (child.startsWith(nimbus.toHostPortString())) { diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java index f1f9473958..6bbe850b0c 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/LocalFsBlobStoreSynchronizerTest.java @@ -31,6 +31,7 @@ import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFacto import org.apache.storm.shade.org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.storm.utils.Utils; import org.apache.storm.testing.InProcessZookeeper; +import org.apache.storm.zookeeper.curator.CuratorZkClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -117,7 +118,7 @@ public class LocalFsBlobStoreSynchronizerTest { // Creating nimbus hosts containing the latest version of blob zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1"); zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2"); - Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1"); + Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(CuratorZkClient.wrap(zkClient), "key1"); assertEquals("nimbus2", (set.iterator().next()).getHost(), "Failed to get the correct nimbus hosts with latest blob version"); zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1"); zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2"); diff --git a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java index 19cdfed25d..2360935c70 100644 --- a/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java +++ b/storm-server/src/test/java/org/apache/storm/blobstore/MockZookeeperClientBuilder.java @@ -15,12 +15,12 @@ package org.apache.storm.blobstore; import java.util.Arrays; import java.util.List; import org.apache.log4j.Logger; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; -import org.apache.storm.shade.org.apache.curator.framework.api.ExistsBuilder; -import org.apache.storm.shade.org.apache.curator.framework.api.GetChildrenBuilder; -import org.apache.storm.shade.org.apache.curator.framework.api.Pathable; import org.apache.storm.shade.org.apache.zookeeper.data.Stat; +import org.apache.storm.zookeeper.ZkClient; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -30,42 +30,31 @@ public class MockZookeeperClientBuilder { private static final Logger LOG = Logger.getLogger(MockZookeeperClientBuilder.class); - private CuratorFramework zkClient = mock(CuratorFramework.class); + private final ZkClient zkClient = mock(ZkClient.class); - private ExistsBuilder existsBuilder = mock(ExistsBuilder.class); - private GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); - - public MockZookeeperClientBuilder() { - when(zkClient.checkExists()).thenReturn(existsBuilder); - when(zkClient.getChildren()).thenReturn(getChildrenBuilder); - } - - public CuratorFramework build() { + public ZkClient build() { return zkClient; } - private <T extends Pathable<U>, U> T mockForPath(T pathable, String path, U toReturn) { + public void withExists(String path, boolean returnValue) { + Stat stat = returnValue ? new Stat() : null; try { - when(pathable.forPath(path)).thenReturn(toReturn); + when(zkClient.exists(eq(path), anyBoolean())).thenReturn(stat); } catch (Exception e) { LOG.warn(e.toString()); } - return pathable; } - public ExistsBuilder withExists(String path, boolean returnValue) { - when(existsBuilder.watched()).thenReturn(existsBuilder); - Stat stat = returnValue ? new Stat() : null; - return mockForPath(existsBuilder, path, stat); + public void withGetChildren(String path, String... returnValue) { + withGetChildren(path, Arrays.asList(returnValue)); } - public GetChildrenBuilder withGetChildren(String path, String... returnValue) { - return withGetChildren(path, Arrays.asList(returnValue)); - } - - public GetChildrenBuilder withGetChildren(String path, List<String> returnValue) { - when(getChildrenBuilder.watched()).thenReturn(getChildrenBuilder); - return mockForPath(getChildrenBuilder, path, returnValue); + public void withGetChildren(String path, List<String> returnValue) { + try { + when(zkClient.getChildren(eq(path), anyBoolean())).thenReturn(returnValue); + } catch (Exception e) { + LOG.warn(e.toString()); + } } public void verifyExists() { @@ -77,7 +66,11 @@ public class MockZookeeperClientBuilder { } public void verifyExists(int times) { - verify(zkClient, times(times)).checkExists(); + try { + verify(zkClient, times(times)).exists(anyString(), anyBoolean()); + } catch (Exception e) { + throw new AssertionError(e); + } } public void verifyGetChildren() { @@ -89,6 +82,10 @@ public class MockZookeeperClientBuilder { } public void verifyGetChildren(int times) { - verify(zkClient, times(times)).getChildren(); + try { + verify(zkClient, times(times)).getChildren(anyString(), anyBoolean()); + } catch (Exception e) { + throw new AssertionError(e); + } } } diff --git a/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java b/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java index fff2fdf4bb..fff8b399c4 100644 --- a/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java +++ b/storm-server/src/test/java/org/apache/storm/cluster/ClusterStateTest.java @@ -43,19 +43,16 @@ import org.apache.storm.shade.org.apache.zookeeper.Watcher; import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; import org.apache.storm.shade.org.apache.zookeeper.data.ACL; import org.apache.storm.testing.InProcessZookeeper; -import org.apache.storm.utils.CuratorUtils; import org.apache.storm.utils.Time; import org.apache.storm.utils.Utils; import org.apache.storm.utils.ZookeeperAuthInfo; -import org.apache.storm.shade.org.apache.curator.framework.CuratorFramework; import org.apache.storm.shade.org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.storm.callback.WatcherCallBack; -import org.apache.storm.shade.org.apache.curator.framework.api.BackgroundVersionable; -import org.apache.storm.shade.org.apache.curator.framework.api.DeleteBuilder; -import org.apache.storm.shade.org.apache.curator.framework.api.ExistsBuilder; import org.apache.storm.shade.org.apache.zookeeper.KeeperException; import org.apache.storm.shade.org.apache.zookeeper.data.Stat; import org.apache.storm.zookeeper.ClientZookeeper; +import org.apache.storm.zookeeper.ZkClient; +import org.apache.storm.zookeeper.curator.CuratorUtils; import org.junit.jupiter.api.Test; import org.awaitility.Awaitility; import org.mockito.Mockito; @@ -551,21 +548,13 @@ public class ClusterStateTest { public void testDeleteNodeSwallowsConcurrentDelete() throws Exception { // Simulates the race where the node is present at the exists-check but removed by another // client before the delete runs: deleteNode must swallow NoNodeException, not rethrow it. - CuratorFramework zk = Mockito.mock(CuratorFramework.class); - - ExistsBuilder existsBuilder = Mockito.mock(ExistsBuilder.class); - Mockito.when(zk.checkExists()).thenReturn(existsBuilder); - Mockito.when(existsBuilder.forPath("/race")).thenReturn(new Stat()); - - DeleteBuilder deleteBuilder = Mockito.mock(DeleteBuilder.class); - BackgroundVersionable childrenDeletable = Mockito.mock(BackgroundVersionable.class); - Mockito.when(zk.delete()).thenReturn(deleteBuilder); - Mockito.when(deleteBuilder.deletingChildrenIfNeeded()).thenReturn(childrenDeletable); - Mockito.when(childrenDeletable.forPath("/race")).thenThrow(new KeeperException.NoNodeException()); + ZkClient zk = Mockito.mock(ZkClient.class); + Mockito.when(zk.exists("/race", false)).thenReturn(new Stat()); + Mockito.doThrow(new KeeperException.NoNodeException()).when(zk).delete("/race", true); assertDoesNotThrow(() -> ClientZookeeper.deleteNode(zk, "/race")); // ensure to delete was actually attempted (the catch branch was exercised, not skipped) - Mockito.verify(deleteBuilder).deletingChildrenIfNeeded(); + Mockito.verify(zk).delete("/race", true); } @Test @@ -580,7 +569,7 @@ public class ClusterStateTest { } }; - CuratorFramework client = ClientZookeeper.mkClient(conf, List.of("localhost"), + ZkClient client = ClientZookeeper.mkClient(conf, List.of("localhost"), zk.getPort(), "", watcher, null, DaemonType.UNKNOWN); try { final String path = String.format("/node-%s", Time.currentTimeMillis()); diff --git a/storm-server/src/test/java/org/apache/storm/zookeeper/AclEnforcementTest.java b/storm-server/src/test/java/org/apache/storm/zookeeper/AclEnforcementTest.java new file mode 100644 index 0000000000..940c7aeccb --- /dev/null +++ b/storm-server/src/test/java/org/apache/storm/zookeeper/AclEnforcementTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.storm.zookeeper; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.security.auth.Subject; +import org.apache.storm.Config; +import org.apache.storm.DaemonConfig; +import org.apache.storm.blobstore.AtomicOutputStream; +import org.apache.storm.blobstore.BlobStore; +import org.apache.storm.blobstore.InputStreamWithMeta; +import org.apache.storm.cluster.DaemonType; +import org.apache.storm.generated.KeyNotFoundException; +import org.apache.storm.generated.ReadableBlobMeta; +import org.apache.storm.generated.SettableBlobMeta; +import org.apache.storm.nimbus.ILeaderElector; +import org.apache.storm.nimbus.NimbusInfo; +import org.apache.storm.shade.org.apache.zookeeper.CreateMode; +import org.apache.storm.shade.org.apache.zookeeper.ZooDefs; +import org.apache.storm.shade.org.apache.zookeeper.data.ACL; +import org.apache.storm.shade.org.apache.zookeeper.data.Id; +import org.apache.storm.shade.org.apache.zookeeper.server.auth.DigestAuthenticationProvider; +import org.apache.storm.testing.InProcessZookeeper; +import org.apache.storm.utils.Utils; +import org.junit.jupiter.api.Test; + +/** + * Exercises {@link AclEnforcement} against a real (in-process) ZooKeeper. + */ +public class AclEnforcementTest { + private static final String LIVE_TOPO = "live-topo-1-1"; + private static final String DEAD_TOPO = "dead-topo-1-1"; + private static final String TOPO_PAYLOAD = "topo:secret"; + + /** + * Minimal blob store that only serves the topology configuration of {@link #LIVE_TOPO}. + */ + public static class StubBlobStore extends BlobStore { + @Override + public void prepare(Map<String, Object> conf, String baseDir, NimbusInfo nimbusInfo, + ILeaderElector leaderElector) { + } + + @Override + public byte[] readBlob(String key, Subject who) throws KeyNotFoundException { + if ((LIVE_TOPO + "-stormconf.ser").equals(key)) { + return Utils.toCompressedJsonConf( + Collections.singletonMap(Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD, TOPO_PAYLOAD)); + } + throw new KeyNotFoundException(key); + } + + @Override + public AtomicOutputStream createBlob(String key, SettableBlobMeta meta, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public AtomicOutputStream updateBlob(String key, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public ReadableBlobMeta getBlobMeta(String key, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public void setLeaderElector(ILeaderElector leaderElector) { + } + + @Override + public void setBlobMeta(String key, SettableBlobMeta meta, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteBlob(String key, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public InputStreamWithMeta getBlob(String key, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public Iterator<String> listKeys() { + return Collections.emptyIterator(); + } + + @Override + public int getBlobReplication(String key, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public int updateBlobReplication(String key, int replication, Subject who) { + throw new UnsupportedOperationException(); + } + + @Override + public void shutdown() { + } + } + + @Test + public void verifyAndFixAcls() throws Exception { + try (InProcessZookeeper zkServer = new InProcessZookeeper()) { + Map<String, Object> conf = Utils.readStormConfig(); + conf.put(Config.STORM_ZOOKEEPER_PORT, zkServer.getPort()); + conf.put(Config.STORM_ZOOKEEPER_SERVERS, Collections.singletonList("localhost")); + conf.put(Config.STORM_ZOOKEEPER_AUTH_SCHEME, "digest"); + conf.put(Config.STORM_ZOOKEEPER_AUTH_PAYLOAD, "storm:password"); + conf.put(Config.STORM_ZOOKEEPER_SUPERACL, + "digest:" + DigestAuthenticationProvider.generateDigest("storm:password")); + conf.put(DaemonConfig.NIMBUS_BLOBSTORE, StubBlobStore.class.getName()); + String root = (String) conf.get(Config.STORM_ZOOKEEPER_ROOT); + + ACL superAcl = Utils.getSuperUserAcl(conf); + Id topoId = new Id("digest", DigestAuthenticationProvider.generateDigest(TOPO_PAYLOAD)); + List<ACL> open = ZooDefs.Ids.OPEN_ACL_UNSAFE; + + try (ZkClient zk = ClientZookeeper.mkClient(conf, Collections.singletonList("localhost"), + zkServer.getPort(), "", null, conf, DaemonType.UNKNOWN)) { + for (String path : Arrays.asList(root, root + "/blobstore", root + "/blobstore/key1", + root + "/storms", root + "/storms/" + LIVE_TOPO, + root + "/credentials", root + "/credentials/" + DEAD_TOPO, + root + "/errors")) { + zk.create(path, new byte[]{1}, CreateMode.PERSISTENT, open, false); + } + + // Wrong ACLs and no fixing must fail. + assertThrows(IllegalStateException.class, () -> AclEnforcement.verifyAcls(conf, false)); + + AclEnforcement.verifyAcls(conf, true); + + Set<ACL> superOnly = Collections.singleton(superAcl); + assertEquals(superOnly, new HashSet<>(zk.getAcl(root))); + assertEquals(superOnly, new HashSet<>(zk.getAcl(root + "/blobstore"))); + assertEquals(superOnly, new HashSet<>(zk.getAcl(root + "/blobstore/key1"))); + assertEquals(superOnly, new HashSet<>(zk.getAcl(root + "/storms"))); + assertEquals(new HashSet<>(Arrays.asList(superAcl, new ACL(ZooDefs.Perms.READ, topoId))), + new HashSet<>(zk.getAcl(root + "/storms/" + LIVE_TOPO))); + assertEquals(superOnly, new HashSet<>(zk.getAcl(root + "/credentials"))); + + // The dead topology's credentials are removed. + assertNull(zk.exists(root + "/credentials/" + DEAD_TOPO, false)); + + // The missing errors node for the live topology is created with its read-write ACL. + assertNotNull(zk.exists(root + "/errors/" + LIVE_TOPO, false)); + assertEquals(new HashSet<>(Arrays.asList(superAcl, new ACL(ZooDefs.Perms.ALL, topoId))), + new HashSet<>(zk.getAcl(root + "/errors/" + LIVE_TOPO))); + + // Everything is now in order. + assertDoesNotThrow(() -> AclEnforcement.verifyAcls(conf, false)); + } + } + } +}
