Can a CRUD operation succeed when we have a ConnectionLossException?

[Benjamin Jaton]

Hi,

In a small test class I keep creating/deleting the same node /test.

I kill (Ctrl+C) and restart the ZK server in the command line randomly.

After a couple attempts here is what we get:















*Fri Oct 02 09:27:23 PDT 2015 - Deleting /testFri Oct 02 09:27:23 PDT 2015
- /test has been deleted.Fri Oct 02 09:27:23 PDT 2015 - Node /test has
successfully been removed.Fri Oct 02 09:27:23 PDT 2015 - Recreating
/testFri Oct 02 09:27:23 PDT 2015 - CONN LOST on createFri Oct 02 09:27:24
PDT 2015 - ERROR: node should be removed.Fri Oct 02 09:27:24 PDT 2015 -
Data of node is: testNode has been mysteriously
created...org.apache.zookeeper.KeeperException$NodeExistsException:
KeeperErrorCode = NodeExists for /test    at
org.apache.zookeeper.KeeperException.create(KeeperException.java:123)    at
org.apache.zookeeper.KeeperException.create(KeeperException.java:51)    at
org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:1209)    at
TestZKSaveConnLoss.create(TestZKSaveConnLoss.java:101)    at
TestZKSaveConnLoss.access$4(TestZKSaveConnLoss.java:98)    at
TestZKSaveConnLoss$2.run(TestZKSaveConnLoss.java:56)*

The 1st create fails with a ConnectionLossException.
When we retry to create the node, we get a NodeExistsException.

So the node got created despite the ConnectionLossException.
The same thing happens with the delete operation.

I was wondering if this is an known/expected behavior and how people deal
with it?

Thanks,
Ben


Original issue reported at:
https://issues.apache.org/jira/browse/CURATOR-268

import java.util.Date;
import java.util.concurrent.CountDownLatch;

import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;

public class TestZKSaveConnLoss {
	static ZooKeeper zooKeeper;

	private final static String path = "/test";
	private final static String dataStr = "test";
	static Thread clientThread;

	public static void main(String[] args) throws Exception {
		final CountDownLatch latch = new CountDownLatch(1);
		Watcher watcher = new Watcher() {
			@Override
			public void process(WatchedEvent event) {
				if (event.getState() == Event.KeeperState.SyncConnected) {
					latch.countDown();
				}
			}
		};
		zooKeeper = new ZooKeeper("localhost:2181", 30 * 1000, watcher);
		latch.await();
		log("starting");
		start();

		Thread.sleep(100000);
	}

	private static void start() {
		clientThread = new Thread() {
			public void run() {
				while (true) {
					try {
						if (checkExists() != null) {
							log("Deleting " + path);
							delete();
							log(path + " has been deleted.");
						}
						if (checkExists() != null) {
							fail("ZK node still here!", null);
						} else {
							log("Node " + path
									+ " has successfully been removed.");
						}
						log("Recreating " + path);
						try {
							create();
						} catch (NodeExistsException e) {
							log("ERROR: node should be removed.");
							byte[] data = getData();
							log("Data of node is: " + new String(data));
							fail("Node has been mysteriously created...", e);
						}
						log(path + " has been created.");
						byte[] data = getData();
						String content = new String(data);
						if (!dataStr.equals(content)) {
							fail("Data doesn't match: " + content, null);
						} else {
							log("Data has been verified.");
						}
					} catch (Exception e) {
						fail("Client error", e);
					}
				}
			}
		};
		clientThread.start();

		try {
			clientThread.join();
		} catch (InterruptedException e) {
			log("Interrupted.");
			System.exit(1);
		}
	}

	private static byte[] getData() throws Exception {
		for (;;) {
			try {
				return zooKeeper.getData(path, false, null);
			} catch (KeeperException.ConnectionLossException e) {
				log("CONN LOST on getData");
				Thread.sleep(1000);
			}
		}
	}

	private static void create() throws Exception {
		for (;;) {
			try {
				zooKeeper.create(path, dataStr.getBytes(),
						ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
				break;
			} catch (KeeperException.ConnectionLossException e) {
				log("CONN LOST on create");
				Thread.sleep(1000);
			}
		}
	}

	private static void delete() throws Exception {
		for (;;) {
			try {
				zooKeeper.delete(path, -1);
				break;
			} catch (KeeperException.ConnectionLossException e) {
				log("CONN LOST on delete");
				Thread.sleep(1000);
			}
		}
	}

	private static Stat checkExists() throws Exception {
		for (;;) {
			try {
				return zooKeeper.exists(path, false);
			} catch (KeeperException.ConnectionLossException e) {
				log("CONN LOST on exists");
				Thread.sleep(1000);
			}
		}
	}

	private static void log(String msg) {
		System.out.println(new Date() + " - " + msg);
	}

	private static void fail(String msg, Exception e) {
		System.err.println(msg);
		if (e != null) {
			e.printStackTrace();
		}
		System.exit(1);
	}

}