Any idea how to fix it? ts=2023-10-13T08:38:14.786Z caller=main.go:583 level=info msg="Starting Prometheus Server" mode=server version="(version=2.47.1, branch=HEAD, revision=c4d1a8beff37cc004f1dc4ab9d2e73193f51aaeb)" ts=2023-10-13T08:38:14.786Z caller=main.go:588 level=info build_context="(go=go1.21.1, platform=linux/amd64, user=root@4829330363be, date=20231004-10:31:16, tags=netgo,builtinassets,stringlabels)" ts=2023-10-13T08:38:14.786Z caller=main.go:589 level=info host_details="(Linux 5.15.0-1041-azure #48-Ubuntu SMP Tue Jun 20 20:34:08 UTC 2023 x86_64 prometheus-5765644cf4-dczhv (none))" ts=2023-10-13T08:38:14.786Z caller=main.go:590 level=info fd_limits="(soft=1048576, hard=1048576)" ts=2023-10-13T08:38:14.786Z caller=main.go:591 level=info vm_limits="(soft=unlimited, hard=unlimited)" ts=2023-10-13T08:38:14.825Z caller=web.go:566 level=info component=web msg="Start listening for connections" address=0.0.0.0:9290 ts=2023-10-13T08:38:14.826Z caller=main.go:1024 level=info msg="Starting TSDB ..." ts=2023-10-13T08:38:14.827Z caller=tls_config.go:274 level=info component=web msg="Listening on" address=[::]:9290 ts=2023-10-13T08:38:14.827Z caller=tls_config.go:310 level=info component=web msg="TLS is enabled." http2=true address=[::]:9290
ts=2023-10-13T08:38:14.940Z caller=db.go:930 level=info component=tsdb msg="Found and deleted tmp block dir" dir=data/01HCM2ZA20JWZ3SSB4MCY7KWJB.tmp-for-creation ts=2023-10-13T08:38:14.945Z caller=dir_locker.go:77 level=warn component=tsdb msg="A lockfile from a previous execution already existed. It was replaced" file=/prometheus/data/lock ts=2023-10-13T08:38:14.985Z caller=head.go:600 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" ts=2023-10-13T08:38:14.986Z caller=head.go:666 level=error component=tsdb msg="Loading on-disk chunks failed" err="iterate on on-disk chunks: corruption in head chunk file data/chunks_head/000001: head chunk file has some unread data, but doesn't include enough bytes to read the chunk header - required:131078, available:131072, file:1" ts=2023-10-13T08:38:14.986Z caller=head.go:906 level=info component=tsdb msg="Deleting mmapped chunk files" ts=2023-10-13T08:38:14.988Z caller=head.go:912 level=info component=tsdb msg="Deleting mmapped chunk files" ts=2023-10-13T08:38:15.000Z caller=head.go:922 level=info component=tsdb msg="Deletion of mmap chunk files successful, reattempting m-mapping the on-disk chunks" ts=2023-10-13T08:38:15.000Z caller=head.go:681 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=14.995821ms ts=2023-10-13T08:38:15.000Z caller=head.go:689 level=info component=tsdb msg="Replaying WAL, this may take a while" ts=2023-10-13T08:38:15.251Z caller=db.go:898 level=warn component=tsdb msg="Encountered WAL read error, attempting repair" err="read records: corruption in segment data/wal/00000000 at 1212416: unexpected non-zero byte in padded page" ts=2023-10-13T08:38:15.252Z caller=wlog.go:426 level=warn component=tsdb msg="Starting corruption repair" segment=0 offset=1212416 ts=2023-10-13T08:38:15.259Z caller=wlog.go:434 level=warn component=tsdb msg="Deleting all segments newer than corrupted segment" segment=0 ts=2023-10-13T08:38:15.279Z caller=wlog.go:456 level=warn component=tsdb msg="Rewrite corrupted segment" segment=0 ts=2023-10-13T08:38:23.386Z caller=db.go:902 level=info component=tsdb msg="Successfully repaired WAL" ts=2023-10-13T08:38:23.391Z caller=main.go:1045 level=info fs_type=fe534d42 ts=2023-10-13T08:38:23.391Z caller=main.go:1048 level=info msg="TSDB started" ts=2023-10-13T08:38:43.281Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM306H4R5JVAP85K42A11C7.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:43.281Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:44.734Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM307SSQPBMGTYZ4VK7D117.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:44.734Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:47.040Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30A6F8GEXNYT4HJC8TN88.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:47.040Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:51.483Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30ED1P7AKQJ7M6A9D5GR2.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:51.483Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:38:59.826Z caller=compact.go:562 level=error component=tsdb msg="removed tmp folder after failed compaction" err="unlinkat data/01HCM30PMVZ1AJJ03HQYYNX6RJ.tmp-for-creation: directory not empty" ts=2023-10-13T08:38:59.827Z caller=db.go:984 level=error component=tsdb msg="compaction failed" err="compact head: persist head block: 2 errors: populate block: add series: read symbols: invalid checksum; read symbols: invalid checksum" ts=2023-10-13T08:39:13.910Z caller=stdlib.go:105 level=error component=web caller="http: TLS handshake error from 10.104.60.63:46062" msg=EOF # kubectl get pods -n grafana NAME READY STATUS RESTARTS AGE grafana-5d5c88677b-wvkhm 1/1 Running 0 75m influxdb-df6b77f54-hzzs9 1/1 Running 0 29d prometheus-5765644cf4-dczhv 1/1 Running 1 (11m ago) 166m $ ls -tlr total 20 -rwxrwxrwx 1 root root 0 Oct 13 14:02 lock drwxrwxrwx 2 root root 0 Oct 13 14:02 wal drwxrwxrwx 2 root root 0 Oct 13 14:02 chunks_head -rwxrwxrwx 1 root root 20001 Oct 13 16:38 queries.active drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM306H4R5JVAP85K42A11C7.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM307SSQPBMGTYZ4VK7D117.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30A6F8GEXNYT4HJC8TN88.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30ED1P7AKQJ7M6A9D5GR2.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:38 01HCM30PMVZ1AJJ03HQYYNX6RJ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:39 01HCM316KKGX4ZC2SJV2DPD4YQ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:39 01HCM3266SHYDKQEYR9BH3K2NK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:40 01HCM3415EK8FCQ5DW3SFVE6HE.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:41 01HCM35W4GP5R0GBDYSMQM3VP0.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:42 01HCM37Q2W4SY54V7Y398718TJ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:43 01HCM39J1BW8QEGV3W2M89YGSK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:44 01HCM3BCZEJ9B56XYJZJTME4PK.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:45 01HCM3D7Y3E7AY4Z9XF6HT4WVZ.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:46 01HCM3F2YB9B8C6DGV7T9VXGED.tmp-for-creation drwxrwxrwx 2 root root 0 Oct 13 16:47 01HCM3GXZ1F13VVZQKCHM696EA.tmp-for-creation -- You received this message because you are subscribed to the Google Groups "Prometheus Users" group. To unsubscribe from this group and stop receiving emails from it, send an email to prometheus-users+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/prometheus-users/b556b7ff-a51e-4591-a717-4d5e2fcb8991n%40googlegroups.com.
