jerrypeng opened a new issue #4072: Exception in bookie when Function state enabled URL: https://github.com/apache/pulsar/issues/4072 **Describe the bug** These exception are thrown in bookies when function state is enabled. Seems like there is something wrong with table service ``` 05:26:27.202 [io-write-scheduler-OrderedScheduler-1-0] ERROR org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore - Failed to open rocksdb at dir /streamlio/pulsar/data/bookkeeper/ranges/ranges/000000000000000010/000000000000000001/000000000000000000 org.rocksdb.RocksDBException: While lock file: /streamlio/pulsar/data/bookkeeper/ranges/ranges/000000000000000010/000000000000000001/000000000000000000/current/LOCK: No locks available at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-5.13.3.jar:?] at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[org.rocksdb-rocksdbjni-5.13.3.jar:?] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:323) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openLocalDB(RocksdbKVStore.java:298) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:288) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.init(RocksdbKVStore.java:233) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$initializeLocalStore$5(AbstractStateStoreWithJournal.java:202) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$executeIO$16(AbstractStateStoreWithJournal.java:471) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:111) [com.google.guava-guava-21.0.jar:?] at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:58) [com.google.guava-guava-21.0.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:75) [com.google.guava-guava-21.0.jar:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_212] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_212] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-all-4.1.32.Final.jar:4.1.32.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212] 05:26:27.202 [io-write-scheduler-OrderedScheduler-1-0] WARN org.apache.bookkeeper.stream.storage.impl.sc.StorageContainerRegistryImpl - De-registered StorageContainer ('10') when failed to start java.util.concurrent.CompletionException: org.apache.bookkeeper.statelib.api.exceptions.StateStoreException: org.rocksdb.RocksDBException: While lock file: /streamlio/pulsar/data/bookkeeper/ranges/ranges/000000000000000010/000000000000000001/000000000000000000/current/LOCK: No locks available at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_212] at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_212] at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:943) ~[?:1.8.0_212] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) ~[?:1.8.0_212] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_212] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_212] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$executeIO$16(AbstractStateStoreWithJournal.java:474) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:111) [com.google.guava-guava-21.0.jar:?] at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:58) [com.google.guava-guava-21.0.jar:?] at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:75) [com.google.guava-guava-21.0.jar:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_212] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_212] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_212] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_212] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_212] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-all-4.1.32.Final.jar:4.1.32.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_212] Caused by: org.apache.bookkeeper.statelib.api.exceptions.StateStoreException: org.rocksdb.RocksDBException: While lock file: /streamlio/pulsar/data/bookkeeper/ranges/ranges/000000000000000010/000000000000000001/000000000000000000/current/LOCK: No locks available at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:334) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openLocalDB(RocksdbKVStore.java:298) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:288) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.init(RocksdbKVStore.java:233) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$initializeLocalStore$5(AbstractStateStoreWithJournal.java:202) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$executeIO$16(AbstractStateStoreWithJournal.java:471) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] ... 12 more Caused by: org.rocksdb.RocksDBException: While lock file: /streamlio/pulsar/data/bookkeeper/ranges/ranges/000000000000000010/000000000000000001/000000000000000000/current/LOCK: No locks available at org.rocksdb.RocksDB.open(Native Method) ~[org.rocksdb-rocksdbjni-5.13.3.jar:?] at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[org.rocksdb-rocksdbjni-5.13.3.jar:?] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:323) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openLocalDB(RocksdbKVStore.java:298) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.openRocksdb(RocksdbKVStore.java:288) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.kv.RocksdbKVStore.init(RocksdbKVStore.java:233) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$initializeLocalStore$5(AbstractStateStoreWithJournal.java:202) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] at org.apache.bookkeeper.statelib.impl.journal.AbstractStateStoreWithJournal.lambda$executeIO$16(AbstractStateStoreWithJournal.java:471) ~[org.apache.bookkeeper-statelib-4.9.1-streamlio-2.jar:4.9.1-streamlio-2] ... 12 more ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
