[
https://issues.apache.org/jira/browse/KAFKA-6188?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16922300#comment-16922300
]
jack handey commented on KAFKA-6188:
------------------------------------
[~lindong] [~TeilaRei] I'm using kafka 2.2.0 download from apache.org. and i
have the similar problem as below:
org.apache.kafka.common.errors.KafkaStorageException: Error while deleting
segments for visitlog-0 in dir
/tmp/kafka-logsorg.apache.kafka.common.errors.KafkaStorageException: Error
while deleting segments for visitlog-0 in dir /tmp/kafka-logsCaused by:
java.nio.file.NoSuchFileException:
/tmp/kafka-logs/visitlog-0/00000000000000003838.log at
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at
sun.nio.fs.UnixCopyFile.move(UnixCopyFile.java:409) at
sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:262) at
java.nio.file.Files.move(Files.java:1395) at
org.apache.kafka.common.utils.Utils.atomicMoveWithFallback(Utils.java:805) at
org.apache.kafka.common.record.FileRecords.renameTo(FileRecords.java:224) at
kafka.log.LogSegment.changeFileSuffixes(LogSegment.scala:488) at
kafka.log.Log.asyncDeleteSegment(Log.scala:1924) at
kafka.log.Log.deleteSegment(Log.scala:1909) at
kafka.log.Log.$anonfun$deleteSegments$3(Log.scala:1455) at
kafka.log.Log.$anonfun$deleteSegments$3$adapted(Log.scala:1455) at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55) at
scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at
kafka.log.Log.$anonfun$deleteSegments$2(Log.scala:1455) at
scala.runtime.java8.JFunction0$mcI$sp.apply(JFunction0$mcI$sp.java:23) at
kafka.log.Log.maybeHandleIOException(Log.scala:2013) at
kafka.log.Log.deleteSegments(Log.scala:1446) at
kafka.log.Log.deleteOldSegments(Log.scala:1441) at
kafka.log.Log.deleteRetentionMsBreachedSegments(Log.scala:1519) at
kafka.log.Log.deleteOldSegments(Log.scala:1509) at
kafka.log.LogManager.$anonfun$cleanupLogs$3(LogManager.scala:913) at
kafka.log.LogManager.$anonfun$cleanupLogs$3$adapted(LogManager.scala:910) at
scala.collection.immutable.List.foreach(List.scala:392) at
kafka.log.LogManager.cleanupLogs(LogManager.scala:910) at
kafka.log.LogManager.$anonfun$startup$2(LogManager.scala:395) at
kafka.utils.KafkaScheduler.$anonfun$schedule$2(KafkaScheduler.scala:114) at
kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at
java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748) Suppressed:
java.nio.file.NoSuchFileException:
/tmp/kafka-logs/visitlog-0/00000000000000003838.log ->
/tmp/kafka-logs/visitlog-0/00000000000000003838.log.deleted at
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at
sun.nio.fs.UnixCopyFile.move(UnixCopyFile.java:396) at
sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:262) at
java.nio.file.Files.move(Files.java:1395) at
org.apache.kafka.common.utils.Utils.atomicMoveWithFallback(Utils.java:802) ...
30 more[2019-09-02 04:48:44,391] INFO [ReplicaManager broker=0] Stopping
serving replicas in dir /tmp/kafka-logs
(kafka.server.ReplicaManager)[2019-09-02 04:48:44,395] INFO
[ReplicaFetcherManager on broker 0] Removed fetcher for partitions
Set(__consumer_offsets-22, __consumer_offsets-30, __consumer_offsets-8,
__consumer_offsets-21, __consumer_offsets-4, __consumer_offsets-27,
__consumer_offsets-7, __consumer_offsets-9, __consumer_offsets-46,
__consumer_offsets-25, __consumer_offsets-35, __consumer_offsets-41,
__consumer_offsets-33, __consumer_offsets-23, __consumer_offsets-49,
__consumer_offsets-47, __consumer_offsets-16, __consumer_offsets-28,
__consumer_offsets-31, __consumer_offsets-36, __consumer_offsets-42,
__consumer_offsets-3, __consumer_offsets-18, __consumer_offsets-37,
__consumer_offsets-15, __consumer_offsets-24, systemlog-0,
__consumer_offsets-38, __consumer_offsets-17, __consumer_offsets-48,
__consumer_offsets-19, __consumer_offsets-11, __consumer_offsets-13,
__consumer_offsets-2, __consumer_offsets-43, visitlog-0, __consumer_offsets-6,
__consumer_offsets-14, __consumer_offsets-20, __consumer_offsets-0,
__consumer_offsets-44, __consumer_offsets-39, __consumer_offsets-12,
__consumer_offsets-45, __consumer_offsets-1, __consumer_offsets-5,
__consumer_offsets-26, __consumer_offsets-29, __consumer_offsets-34,
__consumer_offsets-10, __consumer_offsets-32, __consumer_offsets-40)
(kafka.server.ReplicaFetcherManager)[2019-09-02 04:48:44,396] INFO
[ReplicaAlterLogDirsManager on broker 0] Removed fetcher for partitions
Set(__consumer_offsets-22, __consumer_offsets-30, __consumer_offsets-8,
__consumer_offsets-21, __consumer_offsets-4, __consumer_offsets-27,
__consumer_offsets-7, __consumer_offsets-9, __consumer_offsets-46,
__consumer_offsets-25, __consumer_offsets-35, __consumer_offsets-41,
__consumer_offsets-33, __consumer_offsets-23, __consumer_offsets-49,
__consumer_offsets-47, __consumer_offsets-16, __consumer_offsets-28,
__consumer_offsets-31, __consumer_offsets-36, __consumer_offsets-42,
__consumer_offsets-3, __consumer_offsets-18, __consumer_offsets-37,
__consumer_offsets-15, __consumer_offsets-24, systemlog-0,
__consumer_offsets-38, __consumer_offsets-17, __consumer_offsets-48,
__consumer_offsets-19, __consumer_offsets-11, __consumer_offsets-13,
__consumer_offsets-2, __consumer_offsets-43, visitlog-0, __consumer_offsets-6,
__consumer_offsets-14, __consumer_offsets-20, __consumer_offsets-0,
__consumer_offsets-44, __consumer_offsets-39, __consumer_offsets-12,
__consumer_offsets-45, __consumer_offsets-1, __consumer_offsets-5,
__consumer_offsets-26, __consumer_offsets-29, __consumer_offsets-34,
__consumer_offsets-10, __consumer_offsets-32, __consumer_offsets-40)
(kafka.server.ReplicaAlterLogDirsManager)[2019-09-02 04:48:44,495] INFO
[ReplicaManager broker=0] Broker 0 stopped fetcher for partitions
__consumer_offsets-22,__consumer_offsets-30,__consumer_offsets-8,__consumer_offsets-21,__consumer_offsets-4,__consumer_offsets-27,__consumer_offsets-7,__consumer_offsets-9,__consumer_offsets-46,__consumer_offsets-25,__consumer_offsets-35,__consumer_offsets-41,__consumer_offsets-33,__consumer_offsets-23,__consumer_offsets-49,__consumer_offsets-47,__consumer_offsets-16,__consumer_offsets-28,__consumer_offsets-31,__consumer_offsets-36,__consumer_offsets-42,__consumer_offsets-3,__consumer_offsets-18,__consumer_offsets-37,__consumer_offsets-15,__consumer_offsets-24,systemlog-0,__consumer_offsets-38,__consumer_offsets-17,__consumer_offsets-48,__consumer_offsets-19,__consumer_offsets-11,__consumer_offsets-13,__consumer_offsets-2,__consumer_offsets-43,visitlog-0,__consumer_offsets-6,__consumer_offsets-14,__consumer_offsets-20,__consumer_offsets-0,__consumer_offsets-44,__consumer_offsets-39,__consumer_offsets-12,__consumer_offsets-45,__consumer_offsets-1,__consumer_offsets-5,__consumer_offsets-26,__consumer_offsets-29,__consumer_offsets-34,__consumer_offsets-10,__consumer_offsets-32,__consumer_offsets-40
and stopped moving logs for partitions because they are in the failed log
directory /tmp/kafka-logs. (kafka.server.ReplicaManager)[2019-09-02
04:48:44,496] INFO Stopping serving logs in dir /tmp/kafka-logs
(kafka.log.LogManager)[2019-09-02 04:48:44,513] ERROR Shutdown broker because
all log dirs in /tmp/kafka-logs have failed (kafka.log.LogManager).
If the problem is fixed, shall i clone the code from github( branch 2.2 ) and
compiled by myself?
> Broker fails with FATAL Shutdown - log dirs have failed
> -------------------------------------------------------
>
> Key: KAFKA-6188
> URL: https://issues.apache.org/jira/browse/KAFKA-6188
> Project: Kafka
> Issue Type: Bug
> Components: clients, log
> Affects Versions: 1.0.0, 1.0.1
> Environment: Windows 10
> Reporter: Valentina Baljak
> Assignee: Dong Lin
> Priority: Blocker
> Labels: windows
> Attachments: Segments are opened before deletion,
> kafka_2.10-0.10.2.1.zip, output.txt
>
>
> Just started with version 1.0.0 after a 4-5 months of using 0.10.2.1. The
> test environment is very simple, with only one producer and one consumer.
> Initially, everything started fine, stand alone tests worked as expected.
> However, running my code, Kafka clients fail after approximately 10 minutes.
> Kafka won't start after that and it fails with the same error.
> Deleting logs helps to start again, and the same problem occurs.
> Here is the error traceback:
> [2017-11-08 08:21:57,532] INFO Starting log cleanup with a period of 300000
> ms. (kafka.log.LogManager)
> [2017-11-08 08:21:57,548] INFO Starting log flusher with a default period of
> 9223372036854775807 ms. (kafka.log.LogManager)
> [2017-11-08 08:21:57,798] INFO Awaiting socket connections on 0.0.0.0:9092.
> (kafka.network.Acceptor)
> [2017-11-08 08:21:57,813] INFO [SocketServer brokerId=0] Started 1 acceptor
> threads (kafka.network.SocketServer)
> [2017-11-08 08:21:57,829] INFO [ExpirationReaper-0-Produce]: Starting
> (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
> [2017-11-08 08:21:57,845] INFO [ExpirationReaper-0-DeleteRecords]: Starting
> (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
> [2017-11-08 08:21:57,845] INFO [ExpirationReaper-0-Fetch]: Starting
> (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)
> [2017-11-08 08:21:57,845] INFO [LogDirFailureHandler]: Starting
> (kafka.server.ReplicaManager$LogDirFailureHandler)
> [2017-11-08 08:21:57,860] INFO [ReplicaManager broker=0] Stopping serving
> replicas in dir C:\Kafka\kafka_2.12-1.0.0\kafka-logs
> (kafka.server.ReplicaManager)
> [2017-11-08 08:21:57,860] INFO [ReplicaManager broker=0] Partitions are
> offline due to failure on log directory C:\Kafka\kafka_2.12-1.0.0\kafka-logs
> (kafka.server.ReplicaManager)
> [2017-11-08 08:21:57,860] INFO [ReplicaFetcherManager on broker 0] Removed
> fetcher for partitions (kafka.server.ReplicaFetcherManager)
> [2017-11-08 08:21:57,892] INFO [ReplicaManager broker=0] Broker 0 stopped
> fetcher for partitions because they are in the failed log dir
> C:\Kafka\kafka_2.12-1.0.0\kafka-logs (kafka.server.ReplicaManager)
> [2017-11-08 08:21:57,892] INFO Stopping serving logs in dir
> C:\Kafka\kafka_2.12-1.0.0\kafka-logs (kafka.log.LogManager)
> [2017-11-08 08:21:57,892] FATAL Shutdown broker because all log dirs in
> C:\Kafka\kafka_2.12-1.0.0\kafka-logs have failed (kafka.log.LogManager)
--
This message was sent by Atlassian Jira
(v8.3.2#803003)
