[
https://issues.apache.org/jira/browse/KAFKA-13346?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Amit Gupta updated KAFKA-13346:
-------------------------------
Description:
Hello,
We are using Kafka Streams and we observe that some times on some of the hosts
running streams application, Kafka streams instance fails with unexpected
exception. We are running with 40 stream threads per host and 20 hosts in total.
Can some one please help on what can be the root cause here?
|org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
state-store at location .....
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
~[kafka-streams-2.6.0.jar:?]
Caused by: org.rocksdb.RocksDBException: lock :
...../0_468/rocksdb/state-store/LOCK: No locks available
at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
~[kafka-streams-2.6.0.jar:?]
... 15 more
Some times I also see this exception
|
|org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
state-store at location ...../0_433/rocksdb/state-store
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
~[kafka-streams-2.6.0.jar:?]
Caused by: org.rocksdb.RocksDBException: while open a file for lock:
...../0_433/rocksdb/state-store/LOCK: Too many open files
at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
~[kafka-streams-2.6.0.jar:?]
... 15 more|
was:
Hello,
We are using Kafka Streams and we observe that some times on some of the hosts
running streams application, Kafka streams instance fails with unexpected
exception. We are running with 40 stream threads per host and 20 hosts in total.
Can some one please help on what can be the root cause here?
|org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
state-store at location .....
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
~[kafka-streams-2.6.0.jar:?]
Caused by: org.rocksdb.RocksDBException: lock :
...../0_468/rocksdb/state-store/LOCK: No locks available
at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
~[kafka-streams-2.6.0.jar:?]
... 15 more
Some times I also see this exception
|org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
service-routing-configuration-store at location ...../0_433/rocksdb/state-store
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
~[kafka-streams-2.6.0.jar:?]
at
org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
~[kafka-streams-2.6.0.jar:?]
Caused by: org.rocksdb.RocksDBException: while open a file for lock:
...../0_433/rocksdb/state-store/LOCK: Too many open files
at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
at
org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
~[kafka-streams-2.6.0.jar:?]
... 15 more|
|
> Kafka Streams fails due to RocksDB Locks Not Available Exception
> ----------------------------------------------------------------
>
> Key: KAFKA-13346
> URL: https://issues.apache.org/jira/browse/KAFKA-13346
> Project: Kafka
> Issue Type: Bug
> Affects Versions: 2.6.0
> Reporter: Amit Gupta
> Priority: Major
>
> Hello,
> We are using Kafka Streams and we observe that some times on some of the
> hosts running streams application, Kafka streams instance fails with
> unexpected exception. We are running with 40 stream threads per host and 20
> hosts in total.
> Can some one please help on what can be the root cause here?
>
> |org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
> state-store at location .....
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
> ~[kafka-streams-2.6.0.jar:?]
> Caused by: org.rocksdb.RocksDBException: lock :
> ...../0_468/rocksdb/state-store/LOCK: No locks available
> at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
> at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
> ~[kafka-streams-2.6.0.jar:?]
> ... 15 more
>
> Some times I also see this exception
> |
> |org.apache.kafka.streams.errors.ProcessorStateException: Error opening store
> state-store at location ...../0_433/rocksdb/state-store
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:214)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openDB(RocksDBStore.java:188)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.init(RocksDBStore.java:224)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.ChangeLoggingKeyValueBytesStore.init(ChangeLoggingKeyValueBytesStore.java:42)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.WrappedStateStore.init(WrappedStateStore.java:48)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.MeteredKeyValueStore.lambda$init$0(MeteredKeyValueStore.java:101)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.maybeMeasureLatency(StreamsMetricsImpl.java:836)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.state.internals.MeteredKeyValueStore.init(MeteredKeyValueStore.java:101)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.ProcessorStateManager.registerStateStores(ProcessorStateManager.java:199)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StateManagerUtil.registerStateStores(StateManagerUtil.java:76)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StandbyTask.initializeIfNeeded(StandbyTask.java:95)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.TaskManager.tryToCompleteRestoration(TaskManager.java:426)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:660)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:551)
> ~[kafka-streams-2.6.0.jar:?]
> at
> org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:510)
> ~[kafka-streams-2.6.0.jar:?]
> Caused by: org.rocksdb.RocksDBException: while open a file for lock:
> ...../0_433/rocksdb/state-store/LOCK: Too many open files
> at org.rocksdb.RocksDB.open(Native Method) ~[rocksdbjni-5.18.3.jar:?]
> at org.rocksdb.RocksDB.open(RocksDB.java:286) ~[rocksdbjni-5.18.3.jar:?]
> at
> org.apache.kafka.streams.state.internals.RocksDBStore.openRocksDB(RocksDBStore.java:211)
> ~[kafka-streams-2.6.0.jar:?]
> ... 15 more|
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
