prometheus logs
Last 180 lines from Docker.
2026-11-12T11:32:49.572523749Z time=2026-11-12T11:32:49.572Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:32:53.740808580Z time=2026-11-12T11:32:53.740Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:32:55.893415513Z time=2026-11-12T11:32:55.893Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:02.893902057Z time=2026-11-12T11:33:02.893Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:08.786761545Z time=2026-11-12T11:33:08.786Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:10.928038525Z time=2026-11-12T11:33:10.921Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:17.508400365Z time=2026-11-12T11:33:17.508Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:23.903286716Z time=2026-11-12T11:33:23.902Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:25.932600673Z time=2026-11-12T11:33:25.931Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:33.270631058Z time=2026-11-12T11:33:33.266Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:38.841272568Z time=2026-11-12T11:33:38.841Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:40.892610361Z time=2026-11-12T11:33:40.892Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:47.964041911Z time=2026-11-12T11:33:47.963Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794312000088 maxt=1794319200000 ulid=01M77RJ39VQK0CEHTDA73Z2KPC
2026-11-12T11:33:47.968020133Z time=2026-11-12T11:33:47.967Z level=ERROR source=db.go:1160 msg="compaction failed" component=tsdb err="mkdir /prometheus/01M77RJ39VQK0CEHTDA73Z2KPC.tmp-for-creation: no space left on device"
2026-11-12T11:33:48.716674346Z time=2026-11-12T11:33:48.716Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:53.775632529Z time=2026-11-12T11:33:53.775Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:33:55.934583941Z time=2026-11-12T11:33:55.932Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:03.451657848Z time=2026-11-12T11:34:03.449Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:08.754658753Z time=2026-11-12T11:34:08.754Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:10.890603677Z time=2026-11-12T11:34:10.889Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:20.084557128Z time=2026-11-12T11:34:20.084Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:23.756552228Z time=2026-11-12T11:34:23.756Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:25.890647654Z time=2026-11-12T11:34:25.890Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:33.558639632Z time=2026-11-12T11:34:33.557Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:38.767794873Z time=2026-11-12T11:34:38.767Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:40.907909645Z time=2026-11-12T11:34:40.907Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:47.988768345Z time=2026-11-12T11:34:47.972Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794312000088 maxt=1794319200000 ulid=01M77RKXX4H83YNZHR5TSMV5T9
2026-11-12T11:34:47.988807529Z time=2026-11-12T11:34:47.972Z level=ERROR source=db.go:1160 msg="compaction failed" component=tsdb err="mkdir /prometheus/01M77RKXX4H83YNZHR5TSMV5T9.tmp-for-creation: no space left on device"
2026-11-12T11:34:48.931991089Z time=2026-11-12T11:34:48.931Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:53.767739067Z time=2026-11-12T11:34:53.766Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:34:55.891564756Z time=2026-11-12T11:34:55.890Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:04.418907946Z time=2026-11-12T11:35:04.418Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:08.886269795Z time=2026-11-12T11:35:08.885Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:10.904668521Z time=2026-11-12T11:35:10.904Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:18.208083091Z time=2026-11-12T11:35:18.204Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:23.818737886Z time=2026-11-12T11:35:23.812Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:25.922843131Z time=2026-11-12T11:35:25.922Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:34.199803546Z time=2026-11-12T11:35:34.198Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:38.860808545Z time=2026-11-12T11:35:38.858Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:40.910049812Z time=2026-11-12T11:35:40.909Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:47.976546640Z time=2026-11-12T11:35:47.974Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794312000088 maxt=1794319200000 ulid=01M77RNRG6009D2YBMFB4TKAXP
2026-11-12T11:35:47.976679234Z time=2026-11-12T11:35:47.975Z level=ERROR source=db.go:1160 msg="compaction failed" component=tsdb err="mkdir /prometheus/01M77RNRG6009D2YBMFB4TKAXP.tmp-for-creation: no space left on device"
2026-11-12T11:35:49.745572274Z time=2026-11-12T11:35:49.743Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:53.807561080Z time=2026-11-12T11:35:53.807Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:35:55.928443963Z time=2026-11-12T11:35:55.924Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:04.983526962Z time=2026-11-12T11:36:04.983Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:08.859245676Z time=2026-11-12T11:36:08.858Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:10.907396066Z time=2026-11-12T11:36:10.906Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=prometheus target=http://prometheus:9090/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:19.166431988Z time=2026-11-12T11:36:19.165Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=cadvisor target=http://cadvisor:8080/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:23.846071098Z time=2026-11-12T11:36:23.838Z level=ERROR source=scrape.go:1441 msg="Scrape commit failed" component="scrape manager" scrape_pool=node_exporter target=http://node_exporter:9100/metrics err="write to WAL: log samples: write /prometheus/wal/00003432: no space left on device"
2026-11-12T11:36:44.745060084Z time=2026-11-12T11:36:44.739Z level=WARN source=main.go:1118 msg="Received an OS signal, exiting gracefully..." signal=terminated
2026-11-12T11:36:44.746546384Z time=2026-11-12T11:36:44.746Z level=INFO source=main.go:1143 msg="Stopping scrape discovery manager..."
2026-11-12T11:36:44.749050036Z time=2026-11-12T11:36:44.748Z level=INFO source=main.go:1157 msg="Stopping notify discovery manager..."
2026-11-12T11:36:44.756816089Z time=2026-11-12T11:36:44.754Z level=INFO source=manager.go:216 msg="Stopping rule manager..." component="rule manager"
2026-11-12T11:36:44.756845836Z time=2026-11-12T11:36:44.756Z level=INFO source=manager.go:232 msg="Rule manager stopped" component="rule manager"
2026-11-12T11:36:44.756852208Z time=2026-11-12T11:36:44.756Z level=INFO source=main.go:1194 msg="Stopping scrape manager..."
2026-11-12T11:36:44.757804137Z time=2026-11-12T11:36:44.757Z level=INFO source=main.go:1139 msg="Scrape discovery manager stopped"
2026-11-12T11:36:44.777012386Z time=2026-11-12T11:36:44.776Z level=INFO source=main.go:1153 msg="Notify discovery manager stopped"
2026-11-12T11:36:44.807030600Z time=2026-11-12T11:36:44.805Z level=INFO source=main.go:1186 msg="Scrape manager stopped"
2026-11-12T11:36:47.027848900Z time=2026-11-12T11:36:47.025Z level=INFO source=manager.go:561 msg="Stopping notification manager..." component=notifier
2026-11-12T11:36:47.027905354Z time=2026-11-12T11:36:47.027Z level=INFO source=manager.go:301 msg="Draining any remaining notifications..." component=notifier
2026-11-12T11:36:47.029371372Z time=2026-11-12T11:36:47.027Z level=INFO source=manager.go:307 msg="Remaining notifications drained" component=notifier
2026-11-12T11:36:47.030705451Z time=2026-11-12T11:36:47.029Z level=INFO source=manager.go:234 msg="Notification manager stopped" component=notifier
2026-11-12T11:36:47.030722676Z time=2026-11-12T11:36:47.029Z level=INFO source=main.go:1466 msg="Notifier manager stopped"
2026-11-12T11:36:47.038774228Z time=2026-11-12T11:36:47.038Z level=INFO source=main.go:1480 msg="See you next time!"
2026-11-12T11:36:48.236010354Z time=2026-11-12T11:36:48.232Z level=INFO source=main.go:1589 msg="updated GOGC" old=100 new=75
2026-11-12T11:36:48.241700984Z time=2026-11-12T11:36:48.240Z level=INFO source=main.go:704 msg="Leaving GOMAXPROCS=2: CPU quota undefined" component=automaxprocs
2026-11-12T11:36:48.246584541Z time=2026-11-12T11:36:48.246Z level=INFO source=memlimit.go:198 msg="GOMEMLIMIT is updated" component=automemlimit package=github.com/KimMachineGun/automemlimit/memlimit GOMEMLIMIT=3604897382 previous=9223372036854775807
2026-11-12T11:36:48.246645767Z time=2026-11-12T11:36:48.246Z level=INFO source=main.go:803 msg="Starting Prometheus Server" mode=server version="(version=3.9.1, branch=HEAD, revision=9ec59baffb547e24f1468a53eb82901e58feabd8)"
2026-11-12T11:36:48.246687563Z time=2026-11-12T11:36:48.246Z level=INFO source=main.go:808 msg="operational information" build_context="(go=go1.25.5, platform=linux/amd64, user=root@61c3a9212c9e, date=20260107-16:08:09, tags=netgo,builtinassets)" host_details="(Linux 6.8.0-147-generic #147-Ubuntu SMP PREEMPT_DYNAMIC Thu Sep 24 11:33:16 UTC 2026 x86_64 09ab05bba723 (none))" fd_limits="(soft=1048576, hard=1048576)" vm_limits="(soft=unlimited, hard=unlimited)"
2026-11-12T11:36:48.255594675Z time=2026-11-12T11:36:48.255Z level=INFO source=web.go:684 msg="Start listening for connections" component=web address=0.0.0.0:9090
2026-11-12T11:36:48.256831916Z time=2026-11-12T11:36:48.256Z level=INFO source=main.go:1331 msg="Starting TSDB ..."
2026-11-12T11:36:48.262534205Z time=2026-11-12T11:36:48.262Z level=INFO source=tls_config.go:354 msg="Listening on" component=web address=[::]:9090
2026-11-12T11:36:48.262943111Z time=2026-11-12T11:36:48.262Z level=INFO source=tls_config.go:357 msg="TLS is disabled." component=web http2=false address=[::]:9090
2026-11-12T11:36:48.276635700Z time=2026-11-12T11:36:48.275Z level=INFO source=head.go:681 msg="Replaying on-disk memory mappable chunks if any" component=tsdb
2026-11-12T11:36:48.276662158Z time=2026-11-12T11:36:48.276Z level=INFO source=head.go:767 msg="On-disk memory mappable chunks replay completed" component=tsdb duration=2.966µs
2026-11-12T11:36:48.276763719Z time=2026-11-12T11:36:48.276Z level=INFO source=head.go:775 msg="Replaying WAL, this may take a while" component=tsdb
2026-11-12T11:36:48.277611440Z time=2026-11-12T11:36:48.277Z level=INFO source=head.go:848 msg="WAL segment loaded" component=tsdb segment=0 maxSegment=0 duration=774.288µs
2026-11-12T11:36:48.277628731Z time=2026-11-12T11:36:48.277Z level=INFO source=head.go:885 msg="WAL replay completed" component=tsdb checkpoint_replay_duration=154.469µs wal_replay_duration=823.199µs wbl_replay_duration=151ns chunk_snapshot_load_duration=0s mmap_chunk_replay_duration=2.966µs total_replay_duration=1.023591ms
2026-11-12T11:36:48.285528891Z time=2026-11-12T11:36:48.285Z level=INFO source=main.go:1352 msg="filesystem information" fs_type=EXT4_SUPER_MAGIC
2026-11-12T11:36:48.285558983Z time=2026-11-12T11:36:48.285Z level=INFO source=main.go:1355 msg="TSDB started"
2026-11-12T11:36:48.285563293Z time=2026-11-12T11:36:48.285Z level=INFO source=main.go:1542 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2026-11-12T11:36:48.289932253Z time=2026-11-12T11:36:48.289Z level=INFO source=main.go:1582 msg="Completed loading of configuration file" db_storage=7.84µs remote_storage=15.552µs web_handler=607ns query_engine=1.855µs scrape=1.760059ms scrape_sd=790.432µs notify=2.222µs notify_sd=1.206µs rules=1.953µs tracing=31.654µs filename=/etc/prometheus/prometheus.yml totalDuration=4.339537ms
2026-11-12T11:36:48.289963186Z time=2026-11-12T11:36:48.289Z level=INFO source=main.go:1316 msg="Server is ready to receive web requests."
2026-11-12T11:36:48.290064664Z time=2026-11-12T11:36:48.289Z level=INFO source=manager.go:202 msg="Starting rule manager..." component="rule manager"
2026-11-12T14:33:38.770463153Z time=2026-11-12T14:33:38.758Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794483212638 maxt=1794484800000 ulid=01M782VD62Z0SPKWYEBQK5JGMV
2026-11-12T14:33:39.204937818Z time=2026-11-12T14:33:39.202Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794483212638 maxt=1794484800000 ulid=01M782VD62Z0SPKWYEBQK5JGMV duration=448.41139ms ooo=false
2026-11-12T14:33:39.206771599Z time=2026-11-12T14:33:39.206Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T14:33:39.217580857Z time=2026-11-12T14:33:39.217Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=10.613513ms
2026-11-12T15:00:02.550797293Z time=2026-11-12T15:00:02.543Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794484800303 maxt=1794492000000 ulid=01M784BQVAE8F1MYXFDRPGNE4F
2026-11-12T15:00:02.848420691Z time=2026-11-12T15:00:02.847Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794484800303 maxt=1794492000000 ulid=01M784BQVAE8F1MYXFDRPGNE4F duration=308.851648ms ooo=false
2026-11-12T15:00:02.853138928Z time=2026-11-12T15:00:02.852Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T15:00:02.875272816Z time=2026-11-12T15:00:02.875Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=22.047208ms
2026-11-12T17:00:02.519905687Z time=2026-11-12T17:00:02.512Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794492000042 maxt=1794499200000 ulid=01M78B7F29MZD6AVP2FCJ91NVB
2026-11-12T17:00:02.764826902Z time=2026-11-12T17:00:02.764Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794492000042 maxt=1794499200000 ulid=01M78B7F29MZD6AVP2FCJ91NVB duration=259.271711ms ooo=false
2026-11-12T17:00:02.770502102Z time=2026-11-12T17:00:02.770Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T17:00:02.781775780Z time=2026-11-12T17:00:02.781Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=11.209967ms
2026-11-12T19:00:02.478017208Z time=2026-11-12T19:00:02.467Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794499200231 maxt=1794506400000 ulid=01M78J368ZAYF8P3DCCMDHC23Y
2026-11-12T19:00:02.795080038Z time=2026-11-12T19:00:02.790Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794499200231 maxt=1794506400000 ulid=01M78J368ZAYF8P3DCCMDHC23Y duration=326.960184ms ooo=false
2026-11-12T19:00:02.800019642Z time=2026-11-12T19:00:02.798Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T19:00:02.816831856Z time=2026-11-12T19:00:02.815Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=16.946535ms
2026-11-12T19:00:02.826434584Z time=2026-11-12T19:00:02.824Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=0 to_segment=1 mint=1794506400000
2026-11-12T19:00:03.225643672Z time=2026-11-12T19:00:03.225Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=0 last=1 duration=405.15881ms
2026-11-12T21:00:02.661841610Z time=2026-11-12T21:00:02.651Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794506400005 maxt=1794513600000 ulid=01M78RYXPQ8AS5DGCGB2SJKH65
2026-11-12T21:00:02.949763770Z time=2026-11-12T21:00:02.949Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794506400005 maxt=1794513600000 ulid=01M78RYXPQ8AS5DGCGB2SJKH65 duration=302.359577ms ooo=false
2026-11-12T21:00:02.955337847Z time=2026-11-12T21:00:02.955Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T21:00:02.965543799Z time=2026-11-12T21:00:02.964Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=9.131349ms
2026-11-12T23:00:02.917009951Z time=2026-11-12T23:00:02.916Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794513600326 maxt=1794520800000 ulid=01M78ZTN74701VQ2K6HTAQ77G8
2026-11-12T23:00:03.197568186Z time=2026-11-12T23:00:03.197Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794513600326 maxt=1794520800000 ulid=01M78ZTN74701VQ2K6HTAQ77G8 duration=280.93672ms ooo=false
2026-11-12T23:00:03.199671451Z time=2026-11-12T23:00:03.199Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-12T23:00:03.213319837Z time=2026-11-12T23:00:03.213Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=13.614986ms
2026-11-12T23:00:03.217417919Z time=2026-11-12T23:00:03.216Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=2 to_segment=3 mint=1794520800000
2026-11-12T23:00:03.815067881Z time=2026-11-12T23:00:03.814Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=2 last=3 duration=598.594909ms
2026-11-12T23:00:04.280544046Z time=2026-11-12T23:00:04.279Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=3 mint=1794484800303 maxt=1794506400000 ulid=01M78ZTP375M4DAXSP8H01A264 sources="[01M784BQVAE8F1MYXFDRPGNE4F 01M78B7F29MZD6AVP2FCJ91NVB 01M78J368ZAYF8P3DCCMDHC23Y]" duration=463.778379ms
2026-11-12T23:00:04.297719306Z time=2026-11-12T23:00:04.296Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M78J368ZAYF8P3DCCMDHC23Y
2026-11-12T23:00:04.302734916Z time=2026-11-12T23:00:04.302Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M784BQVAE8F1MYXFDRPGNE4F
2026-11-12T23:00:04.308084942Z time=2026-11-12T23:00:04.307Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M78B7F29MZD6AVP2FCJ91NVB
2026-11-12T23:00:04.772861384Z time=2026-11-12T23:00:04.772Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=2 mint=1794483212638 maxt=1794506400000 ulid=01M78ZTPJM4BSKEM9T7JQBTYFE sources="[01M782VD62Z0SPKWYEBQK5JGMV 01M78ZTP375M4DAXSP8H01A264]" duration=464.000453ms
2026-11-12T23:00:04.780196156Z time=2026-11-12T23:00:04.779Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M782VD62Z0SPKWYEBQK5JGMV
2026-11-12T23:00:04.788654249Z time=2026-11-12T23:00:04.788Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M78ZTP375M4DAXSP8H01A264
2026-11-13T01:00:02.532144977Z time=2026-11-13T01:00:02.525Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794520800135 maxt=1794528000000 ulid=01M796PC2R791YBTG2KP9ZT6AE
2026-11-13T01:00:02.785750895Z time=2026-11-13T01:00:02.785Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794520800135 maxt=1794528000000 ulid=01M796PC2R791YBTG2KP9ZT6AE duration=265.120482ms ooo=false
2026-11-13T01:00:02.790234523Z time=2026-11-13T01:00:02.789Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T01:00:02.801645409Z time=2026-11-13T01:00:02.801Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=11.127659ms
2026-11-13T03:00:03.081657502Z time=2026-11-13T03:00:03.076Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794528000061 maxt=1794535200000 ulid=01M79DJ3W1Z4Q4GXDDZYCVKXEE
2026-11-13T03:00:03.546401511Z time=2026-11-13T03:00:03.545Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794528000061 maxt=1794535200000 ulid=01M79DJ3W1Z4Q4GXDDZYCVKXEE duration=472.319014ms ooo=false
2026-11-13T03:00:03.553648552Z time=2026-11-13T03:00:03.553Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T03:00:03.564655272Z time=2026-11-13T03:00:03.564Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=10.630519ms
2026-11-13T03:00:03.568569261Z time=2026-11-13T03:00:03.567Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=4 to_segment=5 mint=1794535200000
2026-11-13T03:00:04.116195457Z time=2026-11-13T03:00:04.115Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=4 last=5 duration=548.766674ms
2026-11-13T05:00:02.644373731Z time=2026-11-13T05:00:02.636Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794535200036 maxt=1794542400000 ulid=01M79MDTP8AQSC6PGJ4A16GJNV
2026-11-13T05:00:03.221572853Z time=2026-11-13T05:00:03.221Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794535200036 maxt=1794542400000 ulid=01M79MDTP8AQSC6PGJ4A16GJNV duration=588.136525ms ooo=false
2026-11-13T05:00:03.236599300Z time=2026-11-13T05:00:03.233Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T05:00:03.263315768Z time=2026-11-13T05:00:03.263Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=29.966851ms
2026-11-13T05:00:03.794290206Z time=2026-11-13T05:00:03.794Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=3 mint=1794506400005 maxt=1794528000000 ulid=01M79MDVA2EMSVAGHH6DJCMSVM sources="[01M78RYXPQ8AS5DGCGB2SJKH65 01M78ZTN74701VQ2K6HTAQ77G8 01M796PC2R791YBTG2KP9ZT6AE]" duration=527.966401ms
2026-11-13T05:00:03.799117200Z time=2026-11-13T05:00:03.798Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M78RYXPQ8AS5DGCGB2SJKH65
2026-11-13T05:00:03.802818604Z time=2026-11-13T05:00:03.802Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M78ZTN74701VQ2K6HTAQ77G8
2026-11-13T05:00:03.806915724Z time=2026-11-13T05:00:03.806Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M796PC2R791YBTG2KP9ZT6AE
2026-11-13T07:00:02.717635010Z time=2026-11-13T07:00:02.711Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794542400298 maxt=1794549600000 ulid=01M79V9J0M2FPQ8CP6BJY8YSPV
2026-11-13T07:00:03.044731602Z time=2026-11-13T07:00:03.043Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794542400298 maxt=1794549600000 ulid=01M79V9J0M2FPQ8CP6BJY8YSPV duration=335.65619ms ooo=false
2026-11-13T07:00:03.046681175Z time=2026-11-13T07:00:03.046Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T07:00:03.055532683Z time=2026-11-13T07:00:03.055Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=8.840084ms
2026-11-13T07:00:03.059297555Z time=2026-11-13T07:00:03.058Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=6 to_segment=7 mint=1794549600000
2026-11-13T07:00:03.588990155Z time=2026-11-13T07:00:03.588Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=6 last=7 duration=530.638964ms
2026-11-13T09:01:01.506814762Z time=2026-11-13T09:01:00.480Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794549600170 maxt=1794556800000 ulid=01M7A271F62T4N4MHCRAXBKXNF
2026-11-13T09:01:34.876417337Z time=2026-11-13T09:01:31.537Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794549600170 maxt=1794556800000 ulid=01M7A271F62T4N4MHCRAXBKXNF duration=31.221421284s ooo=false
2026-11-13T09:01:37.646987536Z time=2026-11-13T09:01:36.685Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T09:01:38.350618324Z time=2026-11-13T09:01:37.122Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=148.02436ms
2026-11-13T11:09:25.830644089Z time=2026-11-13T11:03:36.461Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794556800241 maxt=1794564000000 ulid=01M7A97DW19ZMPJFB36YMFK0ZR
2026-11-13T11:09:27.927009711Z time=2026-11-13T11:09:27.924Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794556800241 maxt=1794564000000 ulid=01M7A97DW19ZMPJFB36YMFK0ZR duration=5m54.934361102s ooo=false
2026-11-13T11:09:27.955529177Z time=2026-11-13T11:09:27.937Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T11:09:28.105786241Z time=2026-11-13T11:09:28.091Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=153.884786ms
2026-11-13T11:09:28.105813102Z time=2026-11-13T11:09:28.099Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=8 to_segment=9 mint=1794564000000
2026-11-13T11:09:29.395265065Z time=2026-11-13T11:09:29.394Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=8 last=9 duration=1.294635463s
2026-11-13T11:09:30.265357962Z time=2026-11-13T11:09:30.263Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=3 mint=1794528000061 maxt=1794549600000 ulid=01M7A9J9XJPQP5MS30N4FYA0B3 sources="[01M79DJ3W1Z4Q4GXDDZYCVKXEE 01M79MDTP8AQSC6PGJ4A16GJNV 01M79V9J0M2FPQ8CP6BJY8YSPV]" duration=869.124885ms
2026-11-13T11:09:30.276010607Z time=2026-11-13T11:09:30.271Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M79DJ3W1Z4Q4GXDDZYCVKXEE
2026-11-13T11:09:30.276563593Z time=2026-11-13T11:09:30.275Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M79MDTP8AQSC6PGJ4A16GJNV
2026-11-13T11:09:30.286553497Z time=2026-11-13T11:09:30.281Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M79V9J0M2FPQ8CP6BJY8YSPV
2026-11-13T13:00:17.012385060Z time=2026-11-13T13:00:17.005Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794564010942 maxt=1794571200000 ulid=01M7AFX5QB3A5AHPH76QT0M4Q7
2026-11-13T13:00:17.298995746Z time=2026-11-13T13:00:17.298Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794564010942 maxt=1794571200000 ulid=01M7AFX5QB3A5AHPH76QT0M4Q7 duration=294.641862ms ooo=false
2026-11-13T13:00:17.301042651Z time=2026-11-13T13:00:17.300Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T13:00:17.315654929Z time=2026-11-13T13:00:17.314Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=13.8019ms
2026-11-13T15:00:17.015725560Z time=2026-11-13T15:00:17.015Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794571216926 maxt=1794578400000 ulid=01M7APRWZQG49YA8W4B6G67JBT
2026-11-13T15:00:17.284249532Z time=2026-11-13T15:00:17.284Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794571216926 maxt=1794578400000 ulid=01M7APRWZQG49YA8W4B6G67JBT duration=268.733231ms ooo=false
2026-11-13T15:00:17.287212972Z time=2026-11-13T15:00:17.286Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T15:00:17.349459244Z time=2026-11-13T15:00:17.349Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=62.157808ms
2026-11-13T15:00:17.355080483Z time=2026-11-13T15:00:17.353Z level=INFO source=checkpoint.go:100 msg="Creating checkpoint" component=tsdb from_segment=10 to_segment=11 mint=1794578400000
2026-11-13T15:00:17.627590588Z time=2026-11-13T15:00:17.627Z level=INFO source=head.go:1387 msg="WAL checkpoint complete" component=tsdb first=10 last=11 duration=277.111456ms
2026-11-13T17:00:02.173812441Z time=2026-11-13T17:00:02.172Z level=INFO source=compact.go:584 msg="write block started" component=tsdb mint=1794578400338 maxt=1794585600000 ulid=01M7AXM5QWYYWPPT4513KPT5D6
2026-11-13T17:00:02.325580598Z time=2026-11-13T17:00:02.323Z level=INFO source=compact.go:618 msg="write block completed" component=tsdb mint=1794578400338 maxt=1794585600000 ulid=01M7AXM5QWYYWPPT4513KPT5D6 duration=151.55854ms ooo=false
2026-11-13T17:00:02.326842447Z time=2026-11-13T17:00:02.326Z level=INFO source=head.go:1422 msg="Head GC started" component=tsdb caller=truncateMemory
2026-11-13T17:00:02.337976606Z time=2026-11-13T17:00:02.337Z level=INFO source=head.go:1426 msg="Head GC completed" component=tsdb caller=truncateMemory duration=10.9271ms
2026-11-13T17:00:02.796952677Z time=2026-11-13T17:00:02.796Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=3 mint=1794549600170 maxt=1794571200000 ulid=01M7AXM5X3V6CPKSQ3SY17ESGB sources="[01M7A271F62T4N4MHCRAXBKXNF 01M7A97DW19ZMPJFB36YMFK0ZR 01M7AFX5QB3A5AHPH76QT0M4Q7]" duration=456.159137ms
2026-11-13T17:00:02.806606260Z time=2026-11-13T17:00:02.806Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M7A271F62T4N4MHCRAXBKXNF
2026-11-13T17:00:02.810704286Z time=2026-11-13T17:00:02.809Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M7AFX5QB3A5AHPH76QT0M4Q7
2026-11-13T17:00:02.814199069Z time=2026-11-13T17:00:02.813Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M7A97DW19ZMPJFB36YMFK0ZR
2026-11-13T17:00:03.612928017Z time=2026-11-13T17:00:03.612Z level=INFO source=compact.go:556 msg="compact blocks" component=tsdb count=3 mint=1794506400005 maxt=1794571200000 ulid=01M7AXM6BY03VT3N3ETH1BDZCA sources="[01M79MDVA2EMSVAGHH6DJCMSVM 01M7A9J9XJPQP5MS30N4FYA0B3 01M7AXM5X3V6CPKSQ3SY17ESGB]" duration=798.120095ms
2026-11-13T17:00:03.621781554Z time=2026-11-13T17:00:03.621Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M79MDVA2EMSVAGHH6DJCMSVM
2026-11-13T17:00:03.629757543Z time=2026-11-13T17:00:03.629Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M7A9J9XJPQP5MS30N4FYA0B3
2026-11-13T17:00:03.634617902Z time=2026-11-13T17:00:03.634Z level=INFO source=db.go:1921 msg="Deleting obsolete block" component=tsdb block=01M7AXM5X3V6CPKSQ3SY17ESGB