[ https://issues.apache.org/jira/browse/ZOOKEEPER-2383?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15754024#comment-15754024 ]
ASF GitHub Bot commented on ZOOKEEPER-2383: ------------------------------------------- Github user rakeshadr commented on a diff in the pull request: https://github.com/apache/zookeeper/pull/101#discussion_r92785304 --- Diff: src/java/test/org/apache/zookeeper/server/ZooKeeperServerStartupTest.java --- @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.server; + +import static org.apache.zookeeper.client.FourLetterWordMain.send4LetterWord; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.ZKTestCase; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.common.X509Exception.SSLContextException; +import org.apache.zookeeper.test.ClientBase; +import org.apache.zookeeper.test.ClientBase.CountdownWatcher; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class tests the startup behavior of ZooKeeper server. + */ +public class ZooKeeperServerStartupTest extends ZKTestCase { + private static final Logger LOG = LoggerFactory + .getLogger(ZooKeeperServerStartupTest.class); + private static int PORT = PortAssignment.unique(); + private static String HOST = "127.0.0.1"; + private static String HOSTPORT = HOST + ":" + PORT; + private static final String ZK_NOT_SERVING = "This ZooKeeper instance is not currently serving requests"; + + private ServerCnxnFactory servcnxnf; + private ZooKeeperServer zks; + private File tmpDir; + private CountDownLatch startupDelayLatch = new CountDownLatch(1); + + @After + public void teardown() throws Exception { + // count down to avoid infinite blocking call due to this latch, if + // any. + startupDelayLatch.countDown(); + + if (servcnxnf != null) { + servcnxnf.shutdown(); + } + if (zks != null) { + zks.shutdown(); + } + if (zks.getZKDatabase() != null) { + zks.getZKDatabase().close(); + } + ClientBase.recursiveDelete(tmpDir); + } + + /** + * Test case for + * {@link https://issues.apache.org/jira/browse/ZOOKEEPER-2383}. + */ + @Test(timeout = 30000) + public void testClientConnectionRequestDuringStartupWithNIOServerCnxn() + throws Exception { + tmpDir = ClientBase.createTmpDir(); + ClientBase.setupTestEnv(); + + startSimpleZKServer(startupDelayLatch); + SimpleZooKeeperServer simplezks = (SimpleZooKeeperServer) zks; + Assert.assertTrue( + "Failed to invoke zks#startup() method during server startup", + simplezks.waitForStartupInvocation(10)); + + CountdownWatcher watcher = new CountdownWatcher(); + ZooKeeper zkClient = new ZooKeeper(HOSTPORT, + ClientBase.CONNECTION_TIMEOUT, watcher); + + Assert.assertFalse( + "Since server is not fully started, zks#createSession() shouldn't be invoked", + simplezks.waitForSessionCreation(5)); + + LOG.info( + "Decrements the count of the latch, so that server will proceed with startup"); + startupDelayLatch.countDown(); + + Assert.assertTrue("waiting for server being up ", ClientBase + .waitForServerUp(HOSTPORT, ClientBase.CONNECTION_TIMEOUT)); + + Assert.assertTrue( + "Failed to invoke zks#createSession() method during client session creation", + simplezks.waitForSessionCreation(5)); + watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); + zkClient.close(); + } + + /** + * Test case for + * {@link https://issues.apache.org/jira/browse/ZOOKEEPER-2383}. + */ + @Test(timeout = 30000) + public void testClientConnectionRequestDuringStartupWithNettyServerCnxn() + throws Exception { + tmpDir = ClientBase.createTmpDir(); + ClientBase.setupTestEnv(); + + String originalServerCnxnFactory = System + .getProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY); + try { + System.setProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY, + NettyServerCnxnFactory.class.getName()); + startSimpleZKServer(startupDelayLatch); + SimpleZooKeeperServer simplezks = (SimpleZooKeeperServer) zks; + Assert.assertTrue( + "Failed to invoke zks#startup() method during server startup", + simplezks.waitForStartupInvocation(10)); + + CountdownWatcher watcher = new CountdownWatcher(); + ZooKeeper zkClient = new ZooKeeper(HOSTPORT, + ClientBase.CONNECTION_TIMEOUT, watcher); + + Assert.assertFalse( + "Since server is not fully started, zks#createSession() shouldn't be invoked", + simplezks.waitForSessionCreation(5)); + + LOG.info( + "Decrements the count of the latch, so that server will proceed with startup"); + startupDelayLatch.countDown(); + + Assert.assertTrue("waiting for server being up ", ClientBase + .waitForServerUp(HOSTPORT, ClientBase.CONNECTION_TIMEOUT)); + + Assert.assertTrue( + "Failed to invoke zks#createSession() method during client session creation", + simplezks.waitForSessionCreation(5)); + watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT); + zkClient.close(); + } finally { + // reset cnxn factory + if (originalServerCnxnFactory == null) { + System.clearProperty( + ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY); + return; + } + System.setProperty(ServerCnxnFactory.ZOOKEEPER_SERVER_CNXN_FACTORY, + originalServerCnxnFactory); + } + } + + /** + * Test case for + * {@link https://issues.apache.org/jira/browse/ZOOKEEPER-2383}. + */ + @Test(timeout = 30000) + public void testFourLetterWords() throws Exception { + startSimpleZKServer(startupDelayLatch); + verify("conf", ZK_NOT_SERVING); + verify("crst", ZK_NOT_SERVING); + verify("cons", ZK_NOT_SERVING); + verify("dirs", ZK_NOT_SERVING); + verify("dump", ZK_NOT_SERVING); + verify("mntr", ZK_NOT_SERVING); + verify("stat", ZK_NOT_SERVING); + verify("srst", ZK_NOT_SERVING); + verify("wchp", ZK_NOT_SERVING); + verify("wchc", ZK_NOT_SERVING); + verify("wchs", ZK_NOT_SERVING); + verify("isro", "null"); + } + + private void verify(String cmd, String expected) + throws IOException, SSLContextException { + String resp = sendRequest(cmd); + LOG.info("cmd " + cmd + " expected " + expected + " got " + resp); + Assert.assertTrue("Unexpected response", resp.contains(expected)); + } + + private String sendRequest(String cmd) + throws IOException, SSLContextException { + return send4LetterWord(HOST, PORT, cmd); + } + + private void startSimpleZKServer(CountDownLatch startupDelayLatch) + throws IOException { + zks = new SimpleZooKeeperServer(tmpDir, tmpDir, 3000, + startupDelayLatch); + SyncRequestProcessor.setSnapCount(100); + final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]); + + servcnxnf = ServerCnxnFactory.createFactory(PORT, -1); + Thread startupThread = new Thread() { + public void run() { + try { + servcnxnf.startup(zks); + } catch (IOException | InterruptedException e) { --- End diff -- Thank you @fpj . I've separated out the catch block and done another commit in PR. > Startup race in ZooKeeperServer > ------------------------------- > > Key: ZOOKEEPER-2383 > URL: https://issues.apache.org/jira/browse/ZOOKEEPER-2383 > Project: ZooKeeper > Issue Type: Bug > Components: jmx, server > Affects Versions: 3.4.8 > Reporter: Steve Rowe > Assignee: Rakesh R > Priority: Blocker > Fix For: 3.4.10, 3.5.3, 3.6.0 > > Attachments: TestZkStandaloneJMXRegistrationRaceConcurrent.java, > ZOOKEEPER-2383-br-3-4.patch, ZOOKEEPER-2383.patch, ZOOKEEPER-2383.patch, > ZOOKEEPER-2383.patch, ZOOKEEPER-2383.patch, > release-3.4.8-extra-logging.patch, zk-3.4.8-MBeanRegistry.log, > zk-3.4.8-NPE.log > > > In attempting to upgrade Solr's ZooKeeper dependency from 3.4.6 to 3.4.8 > (SOLR-8724) I ran into test failures where attempts to create a node in a > newly started standalone ZooKeeperServer were failing because of an assertion > in MBeanRegistry. > ZooKeeperServer.startup() first sets up its request processor chain then > registers itself in JMX, but if a connection comes in before the server's JMX > registration happens, registration of the connection will fail because it > trips the assertion that (effectively) its parent (the server) has already > registered itself. > {code:java|title=ZooKeeperServer.java} > public synchronized void startup() { > if (sessionTracker == null) { > createSessionTracker(); > } > startSessionTracker(); > setupRequestProcessors(); > registerJMX(); > state = State.RUNNING; > notifyAll(); > } > {code} > {code:java|title=MBeanRegistry.java} > public void register(ZKMBeanInfo bean, ZKMBeanInfo parent) > throws JMException > { > assert bean != null; > String path = null; > if (parent != null) { > path = mapBean2Path.get(parent); > assert path != null; > } > {code} > This problem appears to be new with ZK 3.4.8 - AFAIK Solr never had this > issue with ZK 3.4.6. -- This message was sent by Atlassian JIRA (v6.3.4#6332)