2026-06-10T20:28:19.111034378-04:00 stderr F level=warn ts=2026-06-11T00:28:19.010752573Z caller=loki.go:288 msg="global timeout not configured, using default engine timeout (\"5m0s\"). This behavior will change in the next major to always use the default global timeout (\"5m\")."
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.014803176Z caller=main.go:108 msg="Starting Loki" version="(version=2.9.2, branch=HEAD, revision=a17308db6)"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.017144368Z caller=server.go:322 http=[::]:3100 grpc=[::]:9096 msg="server listening on addresses"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.024667697Z caller=modules.go:1140 msg="-boltdb.shipper.compactor.shared-store not specified, initializing compactor to operator on the following object stores" stores=filesystem
2026-06-10T20:28:19.111034378-04:00 stderr F level=warn ts=2026-06-11T00:28:19.025833742Z caller=cache.go:127 msg="fifocache config is deprecated. use embedded-cache instead"
2026-06-10T20:28:19.111034378-04:00 stderr F level=warn ts=2026-06-11T00:28:19.025935845Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - chunksembedded-cache"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.027510434Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=10.841µs distinct_users_len=0 distinct_users=
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.027588526Z caller=shipper.go:165 index-store=tsdb-2020-10-24 msg="starting index shipper in RW mode"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.028931717Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.02972025Z caller=head_manager.go:299 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.030024819Z caller=manager.go:85 index-store=tsdb-2020-10-24 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.030116123Z caller=head_manager.go:299 index-store=tsdb-2020-10-24 component=tsdb-head-manager msg="loaded wals by period" groups=0
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.036494027Z caller=worker.go:112 msg="Starting querier worker using query-scheduler and scheduler ring for addresses"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.037090725Z caller=mapper.go:47 msg="cleaning up mapped rules directory" path=/tmp/loki/rules-temp
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.04412156Z caller=module_service.go:82 msg=initialising module=server
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.044503331Z caller=module_service.go:82 msg=initialising module=cache-generation-loader
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.044990926Z caller=module_service.go:82 msg=initialising module=query-frontend-tripperware
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.04511973Z caller=module_service.go:82 msg=initialising module=memberlist-kv
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045550003Z caller=module_service.go:82 msg=initialising module=ring
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045644206Z caller=module_service.go:82 msg=initialising module=query-scheduler-ring
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045624835Z caller=module_service.go:82 msg=initialising module=store
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045881603Z caller=ring.go:273 msg="ring doesn't exist in KV store yet"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045891583Z caller=ring.go:273 msg="ring doesn't exist in KV store yet"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.045954395Z caller=basic_lifecycler.go:297 msg="instance not found in the ring" instance=test-loki-785f4d56bc-5mblp ring=scheduler
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046004717Z caller=module_service.go:82 msg=initialising module=ingester-querier
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046009577Z caller=module_service.go:82 msg=initialising module=compactor
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046001127Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046035408Z caller=module_service.go:82 msg=initialising module=ruler
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046083819Z caller=module_service.go:82 msg=initialising module=ingester
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046163541Z caller=ingester.go:431 msg="recovering from checkpoint"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046293275Z caller=ruler.go:528 msg="ruler up and running"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046333127Z caller=ring.go:273 msg="ring doesn't exist in KV store yet"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046418299Z caller=ringmanager.go:201 msg="waiting until scheduler is JOINING in the ring"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046419889Z caller=recovery.go:40 msg="no checkpoint found, treating as no-op"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046411309Z caller=basic_lifecycler.go:297 msg="instance not found in the ring" instance=test-loki-785f4d56bc-5mblp ring=compactor
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046423829Z caller=module_service.go:82 msg=initialising module=distributor
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.04644845Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046523972Z caller=ingester.go:447 msg="recovered WAL checkpoint recovery finished" elapsed=405.362µs errors=false
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046559993Z caller=ingester.go:453 msg="recovering from WAL"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046596434Z caller=basic_lifecycler.go:297 msg="instance not found in the ring" instance=test-loki-785f4d56bc-5mblp ring=distributor
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.046724818Z caller=compactor.go:395 msg="waiting until compactor is JOINING in the ring"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047005948Z caller=ingester.go:469 msg="WAL segment recovery finished" elapsed=879.607µs errors=false
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047039929Z caller=ingester.go:417 msg="closing recoverer"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.04706514Z caller=ingester.go:425 msg="WAL recovery finished" time=946.389µs
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047173343Z caller=lifecycler.go:614 msg="not loading tokens from file, tokens file path is empty"
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047181073Z caller=wal.go:156 msg=started component=wal
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047289186Z caller=lifecycler.go:643 msg="instance not found in ring, adding with no tokens" ring=ingester
2026-06-10T20:28:19.111034378-04:00 stderr F level=info ts=2026-06-11T00:28:19.047442461Z caller=lifecycler.go:483 msg="auto-joining cluster after timeout" ring=ingester
2026-06-10T20:28:19.210905131-04:00 stderr F level=info ts=2026-06-11T00:28:19.184516228Z caller=ringmanager.go:205 msg="scheduler is JOINING in the ring"
2026-06-10T20:28:19.210905131-04:00 stderr F level=info ts=2026-06-11T00:28:19.184695563Z caller=ringmanager.go:214 msg="waiting until scheduler is ACTIVE in the ring"
2026-06-10T20:28:19.310595809-04:00 stderr F level=info ts=2026-06-11T00:28:19.230268521Z caller=compactor.go:399 msg="compactor is JOINING in the ring"
2026-06-10T20:28:19.310595809-04:00 stderr F level=info ts=2026-06-11T00:28:19.230407985Z caller=compactor.go:409 msg="waiting until compactor is ACTIVE in the ring"
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.321824631Z caller=ringmanager.go:218 msg="scheduler is ACTIVE in the ring"
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.321966426Z caller=module_service.go:82 msg=initialising module=query-scheduler
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.322053778Z caller=module_service.go:82 msg=initialising module=query-frontend
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.322163932Z caller=module_service.go:82 msg=initialising module=querier
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.33918536Z caller=compactor.go:413 msg="compactor is ACTIVE in the ring"
2026-06-10T20:28:19.410550814-04:00 stderr F level=info ts=2026-06-11T00:28:19.339240472Z caller=loki.go:505 msg="Loki started"
2026-06-10T20:28:22.410811680-04:00 stderr F level=info ts=2026-06-11T00:28:22.32252773Z caller=scheduler.go:615 msg="this scheduler is in the ReplicationSet, will now accept requests."
2026-06-10T20:28:22.410811680-04:00 stderr F level=info ts=2026-06-11T00:28:22.322569051Z caller=worker.go:209 msg="adding connection" addr=127.0.0.1:9096
2026-06-10T20:28:24.411001245-04:00 stderr F level=info ts=2026-06-11T00:28:24.339579379Z caller=compactor.go:474 msg="this instance has been chosen to run the compactor, starting compactor"
2026-06-10T20:28:24.411001245-04:00 stderr F level=info ts=2026-06-11T00:28:24.339648121Z caller=compactor.go:503 msg="waiting 10m0s for ring to stay stable and previous compactions to finish before starting compactor"
2026-06-10T20:28:29.410665818-04:00 stderr F level=info ts=2026-06-11T00:28:29.322964486Z caller=frontend_scheduler_worker.go:107 msg="adding connection to scheduler" addr=127.0.0.1:9096
2026-06-10T20:29:19.110457833-04:00 stderr F level=info ts=2026-06-11T00:29:19.029749172Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:30:19.110127723-04:00 stderr F level=info ts=2026-06-11T00:30:19.029429811Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:31:19.110901514-04:00 stderr F level=info ts=2026-06-11T00:31:19.029908801Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:32:19.111116222-04:00 stderr F level=info ts=2026-06-11T00:32:19.032695538Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.029098837Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.029142198Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.02919682Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.74µs distinct_users_len=0 distinct_users=
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.047252054Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.047477181Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000000
2026-06-10T20:33:19.111068876-04:00 stderr F level=info ts=2026-06-11T00:33:19.050348409Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000000.tmp new=/tmp/loki/wal/checkpoint.000000
2026-06-10T20:34:19.111165870-04:00 stderr F level=info ts=2026-06-11T00:34:19.029333275Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:35:19.111138536-04:00 stderr F level=info ts=2026-06-11T00:35:19.030085393Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:36:19.110222817-04:00 stderr F level=info ts=2026-06-11T00:36:19.029577638Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:37:19.110987679-04:00 stderr F level=info ts=2026-06-11T00:37:19.029749241Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:38:19.111305421-04:00 stderr F level=info ts=2026-06-11T00:38:19.029143946Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:38:19.111305421-04:00 stderr F level=info ts=2026-06-11T00:38:19.029223378Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:38:19.111305421-04:00 stderr F level=info ts=2026-06-11T00:38:19.029269789Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.66µs distinct_users_len=0 distinct_users=
2026-06-10T20:38:19.111305421-04:00 stderr F level=info ts=2026-06-11T00:38:19.047262993Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:38:19.111305421-04:00 stderr F level=info ts=2026-06-11T00:38:19.047656305Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000001
2026-06-10T20:38:24.411708007-04:00 stderr F level=info ts=2026-06-11T00:38:24.339897269Z caller=compactor.go:508 msg="compactor startup delay completed"
2026-06-10T20:38:24.411708007-04:00 stderr F level=info ts=2026-06-11T00:38:24.339959631Z caller=compactor.go:562 msg="compactor started"
2026-06-10T20:38:24.411708007-04:00 stderr F ts=2026-06-11T00:38:24.340018343Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T20:38:24.411708007-04:00 stderr F ts=2026-06-11T00:38:24.340159797Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=41.611µs
2026-06-10T20:39:19.110920852-04:00 stderr F level=info ts=2026-06-11T00:39:19.029670004Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:40:19.110933846-04:00 stderr F level=info ts=2026-06-11T00:40:19.029449149Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:41:19.110655994-04:00 stderr F level=info ts=2026-06-11T00:41:19.029413345Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:42:19.110798110-04:00 stderr F level=info ts=2026-06-11T00:42:19.029727344Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:42:49.110927167-04:00 stderr F level=info ts=2026-06-11T00:42:49.0578779Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000001.tmp new=/tmp/loki/wal/checkpoint.000001
2026-06-10T20:42:49.110927167-04:00 stderr F level=info ts=2026-06-11T00:42:49.058233171Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.010194414s
2026-06-10T20:43:19.110564997-04:00 stderr F level=info ts=2026-06-11T00:43:19.029563983Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:43:19.110564997-04:00 stderr F level=info ts=2026-06-11T00:43:19.029614135Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:43:19.110564997-04:00 stderr F level=info ts=2026-06-11T00:43:19.029670487Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.441µs distinct_users_len=0 distinct_users=
2026-06-10T20:43:19.110564997-04:00 stderr F level=info ts=2026-06-11T00:43:19.04748365Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:43:19.110564997-04:00 stderr F level=info ts=2026-06-11T00:43:19.048003827Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000002
2026-06-10T20:44:19.110842015-04:00 stderr F level=info ts=2026-06-11T00:44:19.029880043Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:45:19.111261978-04:00 stderr F level=info ts=2026-06-11T00:45:19.029596695Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:46:19.110988469-04:00 stderr F level=info ts=2026-06-11T00:46:19.02944706Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:47:19.111221998-04:00 stderr F level=info ts=2026-06-11T00:47:19.029778562Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:47:49.111978496-04:00 stderr F level=info ts=2026-06-11T00:47:49.061145667Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000002.tmp new=/tmp/loki/wal/checkpoint.000002
2026-06-10T20:47:49.111978496-04:00 stderr F level=info ts=2026-06-11T00:47:49.06157648Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.013232892s
2026-06-10T20:48:19.110328985-04:00 stderr F level=info ts=2026-06-11T00:48:19.029898464Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:48:19.110328985-04:00 stderr F level=info ts=2026-06-11T00:48:19.029963915Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.81µs distinct_users_len=0 distinct_users=
2026-06-10T20:48:19.110328985-04:00 stderr F level=info ts=2026-06-11T00:48:19.030053238Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:48:19.110328985-04:00 stderr F level=info ts=2026-06-11T00:48:19.047237408Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:48:19.110328985-04:00 stderr F level=info ts=2026-06-11T00:48:19.047595329Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000003
2026-06-10T20:48:24.410160096-04:00 stderr F ts=2026-06-11T00:48:24.340900913Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T20:48:24.410160096-04:00 stderr F ts=2026-06-11T00:48:24.341043427Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=47.591µs
2026-06-10T20:49:19.110248164-04:00 stderr F level=info ts=2026-06-11T00:49:19.030149062Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:50:19.110639983-04:00 stderr F level=info ts=2026-06-11T00:50:19.029928204Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:51:19.110722753-04:00 stderr F level=info ts=2026-06-11T00:51:19.029383044Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:52:19.110446561-04:00 stderr F level=info ts=2026-06-11T00:52:19.029114813Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:52:49.110291617-04:00 stderr F level=info ts=2026-06-11T00:52:49.06002664Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000003.tmp new=/tmp/loki/wal/checkpoint.000003
2026-06-10T20:52:49.110291617-04:00 stderr F level=info ts=2026-06-11T00:52:49.060943438Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.012973868s
2026-06-10T20:53:19.113200401-04:00 stderr F level=info ts=2026-06-11T00:53:19.029518636Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:53:19.113200401-04:00 stderr F level=info ts=2026-06-11T00:53:19.029571368Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:53:19.113200401-04:00 stderr F level=info ts=2026-06-11T00:53:19.029631679Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.51µs distinct_users_len=0 distinct_users=
2026-06-10T20:53:19.113200401-04:00 stderr F level=info ts=2026-06-11T00:53:19.047615652Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:53:19.113200401-04:00 stderr F level=info ts=2026-06-11T00:53:19.047935782Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000004
2026-06-10T20:54:19.110459165-04:00 stderr F level=info ts=2026-06-11T00:54:19.03000859Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:55:19.110482811-04:00 stderr F level=info ts=2026-06-11T00:55:19.029775408Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:56:19.110146204-04:00 stderr F level=info ts=2026-06-11T00:56:19.029976748Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:57:19.110735863-04:00 stderr F level=info ts=2026-06-11T00:57:19.029394434Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:57:49.110971917-04:00 stderr F level=info ts=2026-06-11T00:57:49.05897188Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000004.tmp new=/tmp/loki/wal/checkpoint.000004
2026-06-10T20:57:49.110971917-04:00 stderr F level=info ts=2026-06-11T00:57:49.059820435Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.011551273s
2026-06-10T20:58:19.110753147-04:00 stderr F level=info ts=2026-06-11T00:58:19.029253122Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T20:58:19.110753147-04:00 stderr F level=info ts=2026-06-11T00:58:19.029308334Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T20:58:19.110753147-04:00 stderr F level=info ts=2026-06-11T00:58:19.029353815Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.03µs distinct_users_len=0 distinct_users=
2026-06-10T20:58:19.110753147-04:00 stderr F level=info ts=2026-06-11T00:58:19.047231535Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T20:58:19.110753147-04:00 stderr F level=info ts=2026-06-11T00:58:19.047773522Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000005
2026-06-10T20:58:24.411095228-04:00 stderr F ts=2026-06-11T00:58:24.340512509Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T20:58:24.411095228-04:00 stderr F ts=2026-06-11T00:58:24.340636023Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=27.881µs
2026-06-10T20:59:19.110450877-04:00 stderr F level=info ts=2026-06-11T00:59:19.030093267Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:00:19.110362377-04:00 stderr F level=info ts=2026-06-11T01:00:19.030051519Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:01:19.111157078-04:00 stderr F level=info ts=2026-06-11T01:01:19.030076301Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:02:19.110980069-04:00 stderr F level=info ts=2026-06-11T01:02:19.029487779Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:02:49.110603137-04:00 stderr F level=info ts=2026-06-11T01:02:49.096519535Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000005.tmp new=/tmp/loki/wal/checkpoint.000005
2026-06-10T21:02:49.110603137-04:00 stderr F level=info ts=2026-06-11T01:02:49.097564266Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.049430463s
2026-06-10T21:03:19.115015371-04:00 stderr F level=info ts=2026-06-11T01:03:19.029503248Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:03:19.115015371-04:00 stderr F level=info ts=2026-06-11T01:03:19.029552519Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:03:19.115015371-04:00 stderr F level=info ts=2026-06-11T01:03:19.029606691Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.04µs distinct_users_len=0 distinct_users=
2026-06-10T21:03:19.115015371-04:00 stderr F level=info ts=2026-06-11T01:03:19.04752499Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:03:19.115015371-04:00 stderr F level=info ts=2026-06-11T01:03:19.047894841Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000006
2026-06-10T21:04:19.110917590-04:00 stderr F level=info ts=2026-06-11T01:04:19.029703699Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:05:19.110230359-04:00 stderr F level=info ts=2026-06-11T01:05:19.029471632Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:06:19.110421671-04:00 stderr F level=info ts=2026-06-11T01:06:19.036971348Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:07:19.110932738-04:00 stderr F level=info ts=2026-06-11T01:07:19.029764738Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:07:49.110574845-04:00 stderr F level=info ts=2026-06-11T01:07:49.053502705Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000006.tmp new=/tmp/loki/wal/checkpoint.000006
2026-06-10T21:07:49.110574845-04:00 stderr F level=info ts=2026-06-11T01:07:49.054743172Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.00650852s
2026-06-10T21:08:19.111188901-04:00 stderr F level=info ts=2026-06-11T01:08:19.029540887Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:08:19.111188901-04:00 stderr F level=info ts=2026-06-11T01:08:19.029618069Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:08:19.111188901-04:00 stderr F level=info ts=2026-06-11T01:08:19.029691871Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.82µs distinct_users_len=0 distinct_users=
2026-06-10T21:08:19.111188901-04:00 stderr F level=info ts=2026-06-11T01:08:19.04724571Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:08:19.111188901-04:00 stderr F level=info ts=2026-06-11T01:08:19.047635802Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000007
2026-06-10T21:08:24.412099549-04:00 stderr F ts=2026-06-11T01:08:24.340357378Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:08:24.412099549-04:00 stderr F ts=2026-06-11T01:08:24.340510532Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=31.561µs
2026-06-10T21:09:19.110731905-04:00 stderr F level=info ts=2026-06-11T01:09:19.029827593Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:10:19.110476696-04:00 stderr F level=info ts=2026-06-11T01:10:19.029076007Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:11:19.110822549-04:00 stderr F level=info ts=2026-06-11T01:11:19.029918075Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:12:19.111166874-04:00 stderr F level=info ts=2026-06-11T01:12:19.029874038Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:12:49.110935179-04:00 stderr F level=info ts=2026-06-11T01:12:49.055191897Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000007.tmp new=/tmp/loki/wal/checkpoint.000007
2026-06-10T21:12:49.110935179-04:00 stderr F level=info ts=2026-06-11T01:12:49.056345812Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.008320588s
2026-06-10T21:13:19.110442547-04:00 stderr F level=info ts=2026-06-11T01:13:19.029899924Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:13:19.110442547-04:00 stderr F level=info ts=2026-06-11T01:13:19.029964435Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:13:19.110442547-04:00 stderr F level=info ts=2026-06-11T01:13:19.029990286Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.62µs distinct_users_len=0 distinct_users=
2026-06-10T21:13:19.110442547-04:00 stderr F level=info ts=2026-06-11T01:13:19.047344209Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:13:19.110442547-04:00 stderr F level=info ts=2026-06-11T01:13:19.047571316Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000008
2026-06-10T21:14:19.110772503-04:00 stderr F level=info ts=2026-06-11T01:14:19.029719894Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:15:19.111021276-04:00 stderr F level=info ts=2026-06-11T01:15:19.029443051Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:16:19.110941604-04:00 stderr F level=info ts=2026-06-11T01:16:19.029769053Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:17:19.110843322-04:00 stderr F level=info ts=2026-06-11T01:17:19.030091253Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:17:49.110239837-04:00 stderr F level=info ts=2026-06-11T01:17:49.057227749Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000008.tmp new=/tmp/loki/wal/checkpoint.000008
2026-06-10T21:17:49.110239837-04:00 stderr F level=info ts=2026-06-11T01:17:49.058438027Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.0104818s
2026-06-10T21:18:19.111219933-04:00 stderr F level=info ts=2026-06-11T01:18:19.029191034Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:18:19.111219933-04:00 stderr F level=info ts=2026-06-11T01:18:19.029247056Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.74µs distinct_users_len=0 distinct_users=
2026-06-10T21:18:19.111219933-04:00 stderr F level=info ts=2026-06-11T01:18:19.029327288Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:18:19.111219933-04:00 stderr F level=info ts=2026-06-11T01:18:19.047249259Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:18:19.111219933-04:00 stderr F level=info ts=2026-06-11T01:18:19.047561249Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000009
2026-06-10T21:18:24.410724625-04:00 stderr F ts=2026-06-11T01:18:24.341132741Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:18:24.410724625-04:00 stderr F ts=2026-06-11T01:18:24.341266285Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=29.681µs
2026-06-10T21:19:19.111156992-04:00 stderr F level=info ts=2026-06-11T01:19:19.030047234Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:20:19.110506838-04:00 stderr F level=info ts=2026-06-11T01:20:19.029534064Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:21:19.111001575-04:00 stderr F level=info ts=2026-06-11T01:21:19.029110214Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:22:19.110734292-04:00 stderr F level=info ts=2026-06-11T01:22:19.029306545Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:22:49.110228693-04:00 stderr F level=info ts=2026-06-11T01:22:49.059757209Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000009.tmp new=/tmp/loki/wal/checkpoint.000009
2026-06-10T21:22:49.110228693-04:00 stderr F level=info ts=2026-06-11T01:22:49.060633035Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.012568351s
2026-06-10T21:23:19.127915647-04:00 stderr F level=info ts=2026-06-11T01:23:19.02900828Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:23:19.127915647-04:00 stderr F level=info ts=2026-06-11T01:23:19.029068951Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.73µs distinct_users_len=0 distinct_users=
2026-06-10T21:23:19.127915647-04:00 stderr F level=info ts=2026-06-11T01:23:19.029132773Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:23:19.127915647-04:00 stderr F level=info ts=2026-06-11T01:23:19.047275103Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:23:19.127915647-04:00 stderr F level=info ts=2026-06-11T01:23:19.047572722Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000010
2026-06-10T21:24:19.110821031-04:00 stderr F level=info ts=2026-06-11T01:24:19.032014253Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:25:19.111072053-04:00 stderr F level=info ts=2026-06-11T01:25:19.029681377Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:26:19.110349662-04:00 stderr F level=info ts=2026-06-11T01:26:19.029972677Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:27:19.815887356-04:00 stderr F level=info ts=2026-06-11T01:27:19.787075227Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:27:49.110691457-04:00 stderr F level=info ts=2026-06-11T01:27:49.053725126Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000010.tmp new=/tmp/loki/wal/checkpoint.000010
2026-06-10T21:27:49.110691457-04:00 stderr F level=info ts=2026-06-11T01:27:49.054744057Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.006909827s
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.02935923Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.029446523Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.029477994Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=3.21µs distinct_users_len=0 distinct_users=
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.029585927Z caller=table_manager.go:244 index-store=tsdb-2020-10-24 msg="cleaning tables cache"
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.047494235Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:28:19.110442608-04:00 stderr F level=info ts=2026-06-11T01:28:19.047787384Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000011
2026-06-10T21:28:22.410407941-04:00 stderr F level=info ts=2026-06-11T01:28:22.408588556Z caller=roundtrip.go:241 org_id=fake traceID=425515fcf3fd8c6d msg="executing query" type=range query="{application=\"zuul\"}" length=1h0m0s step=14s query_hash=1105929788
2026-06-10T21:28:22.410407941-04:00 stderr F level=info ts=2026-06-11T01:28:22.409678839Z caller=engine.go:234 component=querier org_id=fake traceID=425515fcf3fd8c6d msg="executing query" type=range query="{application=\"zuul\"}" length=28m22.401393995s step=14s query_hash=1105929788
2026-06-10T21:28:22.510585803-04:00 stderr F level=info ts=2026-06-11T01:28:22.413154075Z caller=table_manager.go:195 index-store=tsdb-2020-10-24 msg="downloading all files for table index_20615"
2026-06-10T21:28:22.510585803-04:00 stderr F ts=2026-06-11T01:28:22.413720152Z caller=spanlogger.go:86 table-name=index_20615 org_id=fake traceID=425515fcf3fd8c6d level=info msg="downloaded index set at query time" duration=205.916µs
2026-06-10T21:28:22.510585803-04:00 stderr F ts=2026-06-11T01:28:22.414124925Z caller=spanlogger.go:86 table-name=index_20615 user-id=fake org_id=fake traceID=425515fcf3fd8c6d level=info msg="downloaded index set at query time" duration=139.485µs
2026-06-10T21:28:22.510585803-04:00 stderr F level=info ts=2026-06-11T01:28:22.414734133Z caller=metrics.go:159 component=querier org_id=fake traceID=425515fcf3fd8c6d latency=fast query="{application=\"zuul\"}" query_hash=1105929788 query_type=limited range_type=range length=28m22.401393995s start_delta=28m22.414717543s end_delta=13.323958ms step=14s duration=4.866659ms status=200 limit=5 returned_lines=5 throughput=105MB total_bytes=509kB total_bytes_structured_metadata=0B lines_per_second=316644 total_lines=1541 post_filter_lines=1541 total_entries=5 store_chunks_download_time=0s queue_time=49.352µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:22.510585803-04:00 stderr F level=info ts=2026-06-11T01:28:22.418766166Z caller=metrics.go:159 component=frontend org_id=fake traceID=425515fcf3fd8c6d latency=fast query="{application=\"zuul\"}" query_hash=1105929788 query_type=limited range_type=range length=1h0m0s start_delta=1h0m0.017358311s end_delta=17.358631ms step=14s duration=9.745748ms status=200 limit=5 returned_lines=0 throughput=52MB total_bytes=509kB total_bytes_structured_metadata=0B lines_per_second=158120 total_lines=1541 post_filter_lines=1541 total_entries=5 store_chunks_download_time=0s queue_time=49µs splits=1 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.774103776Z caller=roundtrip.go:241 org_id=fake traceID=7b11652a05e73bbe msg="executing query" type=range query="{application=\"nodepool\"}" length=1h0m0s step=14s query_hash=1297607390
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.775077756Z caller=engine.go:234 component=querier org_id=fake traceID=7b11652a05e73bbe msg="executing query" type=range query="{application=\"nodepool\"}" length=28m22.769593318s step=14s query_hash=1297607390
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.776599173Z caller=metrics.go:159 component=querier org_id=fake traceID=7b11652a05e73bbe latency=fast query="{application=\"nodepool\"}" query_hash=1297607390 query_type=limited range_type=range length=28m22.769593318s start_delta=28m22.776581522s end_delta=6.988564ms step=14s duration=1.379792ms status=200 limit=5 returned_lines=5 throughput=18MB total_bytes=25kB total_bytes_structured_metadata=0B lines_per_second=64502 total_lines=89 post_filter_lines=89 total_entries=5 store_chunks_download_time=0s queue_time=62.572µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.777694216Z caller=metrics.go:159 component=frontend org_id=fake traceID=7b11652a05e73bbe latency=fast query="{application=\"nodepool\"}" query_hash=1297607390 query_type=limited range_type=range length=1h0m0s start_delta=1h0m0.008088907s end_delta=8.089237ms step=14s duration=3.28026ms status=200 limit=5 returned_lines=0 throughput=7.7MB total_bytes=25kB total_bytes_structured_metadata=0B lines_per_second=27131 total_lines=89 post_filter_lines=89 total_entries=5 store_chunks_download_time=0s queue_time=63µs splits=1 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.778182171Z caller=engine.go:234 component=querier org_id=fake traceID=7b11652a05e73bbe msg="executing query" type=range query="{application=\"nodepool\"}" length=30m0s step=14s query_hash=1297607390
2026-06-10T21:28:22.810226810-04:00 stderr F level=error ts=2026-06-11T01:28:22.778978815Z caller=scheduler_processor.go:106 msg="error processing requests from scheduler" err="rpc error: code = Canceled desc = context canceled" addr=127.0.0.1:9096
2026-06-10T21:28:22.810226810-04:00 stderr F level=info ts=2026-06-11T01:28:22.77911593Z caller=metrics.go:159 component=querier org_id=fake traceID=7b11652a05e73bbe latency=fast query="{application=\"nodepool\"}" query_hash=1297607390 query_type=limited range_type=range length=30m0s start_delta=58m22.779105019s end_delta=28m22.779105299s step=14s duration=820.564µs status=500 limit=5 returned_lines=0 throughput=0B total_bytes=0B total_bytes_structured_metadata=0B lines_per_second=0 total_lines=0 post_filter_lines=0 total_entries=0 store_chunks_download_time=0s queue_time=48.842µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:22.810226810-04:00 stderr F level=error ts=2026-06-11T01:28:22.779291325Z caller=scheduler_processor.go:158 org_id=fake traceID=7b11652a05e73bbe msg="error notifying scheduler about finished query" err=EOF addr=127.0.0.1:9096
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.147126896Z caller=roundtrip.go:241 org_id=fake traceID=31fce1b1c8e7286d msg="executing query" type=range query="{application=\"mariadb\"}" length=1h0m0s step=14s query_hash=3115875814
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.148094706Z caller=engine.go:234 component=querier org_id=fake traceID=31fce1b1c8e7286d msg="executing query" type=range query="{application=\"mariadb\"}" length=28m23.142540127s step=14s query_hash=3115875814
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.149303633Z caller=metrics.go:159 component=querier org_id=fake traceID=31fce1b1c8e7286d latency=fast query="{application=\"mariadb\"}" query_hash=3115875814 query_type=limited range_type=range length=28m23.142540127s start_delta=28m23.149294793s end_delta=6.754836ms step=14s duration=1.111293ms status=200 limit=5 returned_lines=3 throughput=500kB total_bytes=556B total_bytes_structured_metadata=0B lines_per_second=2699 total_lines=3 post_filter_lines=3 total_entries=3 store_chunks_download_time=0s queue_time=90.083µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.150507709Z caller=engine.go:234 component=querier org_id=fake traceID=31fce1b1c8e7286d msg="executing query" type=range query="{application=\"mariadb\"}" length=30m0s step=14s query_hash=3115875814
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.151443748Z caller=metrics.go:159 component=querier org_id=fake traceID=31fce1b1c8e7286d latency=fast query="{application=\"mariadb\"}" query_hash=3115875814 query_type=limited range_type=range length=30m0s start_delta=58m23.151437398s end_delta=28m23.151437568s step=14s duration=860.617µs status=200 limit=5 returned_lines=5 throughput=22MB total_bytes=19kB total_bytes_structured_metadata=0B lines_per_second=134787 total_lines=116 post_filter_lines=116 total_entries=5 store_chunks_download_time=0s queue_time=55.531µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:23.210231555-04:00 stderr F level=info ts=2026-06-11T01:28:23.152215742Z caller=metrics.go:159 component=frontend org_id=fake traceID=31fce1b1c8e7286d latency=fast query="{application=\"mariadb\"}" query_hash=3115875814 query_type=limited range_type=range length=1h0m0s start_delta=1h0m0.009659884s end_delta=9.660184ms step=14s duration=4.690543ms status=200 limit=5 returned_lines=0 throughput=4.3MB total_bytes=20kB total_bytes_structured_metadata=0B lines_per_second=25370 total_lines=119 post_filter_lines=119 total_entries=5 store_chunks_download_time=0s queue_time=146µs splits=2 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:23.610678224-04:00 stderr F level=info ts=2026-06-11T01:28:23.524156149Z caller=roundtrip.go:241 org_id=fake traceID=1bd4b817bd6df807 msg="executing query" type=range query="{application=\"zookeeper\"}" length=1h0m0s step=14s query_hash=205527032
2026-06-10T21:28:23.610678224-04:00 stderr F level=info ts=2026-06-11T01:28:23.525326305Z caller=engine.go:234 component=querier org_id=fake traceID=1bd4b817bd6df807 msg="executing query" type=range query="{application=\"zookeeper\"}" length=28m23.518375953s step=14s query_hash=205527032
2026-06-10T21:28:23.610678224-04:00 stderr F level=info ts=2026-06-11T01:28:23.526910703Z caller=metrics.go:159 component=querier org_id=fake traceID=1bd4b817bd6df807 latency=fast query="{application=\"zookeeper\"}" query_hash=205527032 query_type=limited range_type=range length=28m23.518375953s start_delta=28m23.526897463s end_delta=8.52179ms step=14s duration=1.396644ms status=200 limit=5 returned_lines=5 throughput=65MB total_bytes=91kB total_bytes_structured_metadata=0B lines_per_second=281388 total_lines=393 post_filter_lines=393 total_entries=5 store_chunks_download_time=0s queue_time=153.865µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:23.610678224-04:00 stderr F level=info ts=2026-06-11T01:28:23.527814531Z caller=metrics.go:159 component=frontend org_id=fake traceID=1bd4b817bd6df807 latency=fast query="{application=\"zookeeper\"}" query_hash=205527032 query_type=limited range_type=range length=1h0m0s start_delta=1h0m0.009428207s end_delta=9.428437ms step=14s duration=3.397393ms status=200 limit=5 returned_lines=0 throughput=27MB total_bytes=91kB total_bytes_structured_metadata=0B lines_per_second=115676 total_lines=393 post_filter_lines=393 total_entries=5 store_chunks_download_time=0s queue_time=154µs splits=1 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.01141933Z caller=roundtrip.go:241 org_id=fake traceID=5caa699f41254ff9 msg="executing query" type=range query="{labels_run=\"dib\"}" length=1h0m0s step=14s query_hash=2517673860
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.01240995Z caller=engine.go:234 component=querier org_id=fake traceID=5caa699f41254ff9 msg="executing query" type=range query="{labels_run=\"dib\"}" length=28m24.004632962s step=14s query_hash=2517673860
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.013712571Z caller=metrics.go:159 component=querier org_id=fake traceID=5caa699f41254ff9 latency=fast query="{labels_run=\"dib\"}" query_hash=2517673860 query_type=limited range_type=range length=28m24.004632962s start_delta=28m24.01369975s end_delta=9.067078ms step=14s duration=1.125705ms status=200 limit=5 returned_lines=5 throughput=3.8MB total_bytes=4.3kB total_bytes_structured_metadata=0B lines_per_second=16878 total_lines=19 post_filter_lines=19 total_entries=5 store_chunks_download_time=0s queue_time=74.402µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.014841095Z caller=metrics.go:159 component=frontend org_id=fake traceID=5caa699f41254ff9 latency=fast query="{labels_run=\"dib\"}" query_hash=2517673860 query_type=limited range_type=range length=1h0m0s start_delta=1h0m0.010194952s end_delta=10.195232ms step=14s duration=3.124205ms status=200 limit=5 returned_lines=0 throughput=1.4MB total_bytes=4.3kB total_bytes_structured_metadata=0B lines_per_second=6081 total_lines=19 post_filter_lines=19 total_entries=5 store_chunks_download_time=0s queue_time=74µs splits=1 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.015250757Z caller=engine.go:234 component=querier org_id=fake traceID=5caa699f41254ff9 msg="executing query" type=range query="{labels_run=\"dib\"}" length=30m0s step=14s query_hash=2517673860
2026-06-10T21:28:24.110644763-04:00 stderr F level=error ts=2026-06-11T01:28:24.015255187Z caller=scheduler_processor.go:106 msg="error processing requests from scheduler" err="rpc error: code = Canceled desc = context canceled" addr=127.0.0.1:9096
2026-06-10T21:28:24.110644763-04:00 stderr F level=info ts=2026-06-11T01:28:24.015389571Z caller=metrics.go:159 component=querier org_id=fake traceID=5caa699f41254ff9 latency=fast query="{labels_run=\"dib\"}" query_hash=2517673860 query_type=limited range_type=range length=30m0s start_delta=58m24.015381931s end_delta=28m24.015382141s step=14s duration=57.242µs status=400 limit=5 returned_lines=0 throughput=0B total_bytes=0B total_bytes_structured_metadata=0B lines_per_second=0 total_lines=0 post_filter_lines=0 total_entries=0 store_chunks_download_time=0s queue_time=67.812µs splits=0 shards=0 cache_chunk_req=0 cache_chunk_hit=0 cache_chunk_bytes_stored=0 cache_chunk_bytes_fetched=0 cache_chunk_download_time=0s cache_index_req=0 cache_index_hit=0 cache_index_download_time=0s cache_stats_results_req=0 cache_stats_results_hit=0 cache_stats_results_download_time=0s cache_result_req=0 cache_result_hit=0 cache_result_download_time=0s
2026-06-10T21:28:24.110644763-04:00 stderr F level=error ts=2026-06-11T01:28:24.015484254Z caller=scheduler_processor.go:158 org_id=fake traceID=5caa699f41254ff9 msg="error notifying scheduler about finished query" err=EOF addr=127.0.0.1:9096
2026-06-10T21:28:24.411175067-04:00 stderr F ts=2026-06-11T01:28:24.340813717Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:28:24.411175067-04:00 stderr F ts=2026-06-11T01:28:24.340961801Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=29.821µs
2026-06-10T21:29:19.110623097-04:00 stderr F level=info ts=2026-06-11T01:29:19.030009993Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:30:19.110870163-04:00 stderr F level=info ts=2026-06-11T01:30:19.029598719Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:31:19.110756675-04:00 stderr F level=info ts=2026-06-11T01:31:19.029591954Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:32:19.110903238-04:00 stderr F level=info ts=2026-06-11T01:32:19.029772178Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:32:49.110472963-04:00 stderr F level=info ts=2026-06-11T01:32:49.057987988Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000011.tmp new=/tmp/loki/wal/checkpoint.000011
2026-06-10T21:32:49.110472963-04:00 stderr F level=info ts=2026-06-11T01:32:49.059059131Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.010863405s
2026-06-10T21:33:19.110603641-04:00 stderr F level=info ts=2026-06-11T01:33:19.029476511Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:33:19.110603641-04:00 stderr F level=info ts=2026-06-11T01:33:19.029836712Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.029955946Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.030014428Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=25.241µs
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.030029428Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.030046089Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=9.36µs
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.03008239Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:33:19.110603641-04:00 stderr F ts=2026-06-11T01:33:19.03009846Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=9.14µs
2026-06-10T21:33:19.110603641-04:00 stderr F level=info ts=2026-06-11T01:33:19.030120381Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=2.48µs distinct_users_len=0 distinct_users=
2026-06-10T21:33:19.110603641-04:00 stderr F level=info ts=2026-06-11T01:33:19.047236454Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:33:19.110603641-04:00 stderr F level=info ts=2026-06-11T01:33:19.047516952Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000012
2026-06-10T21:34:19.111247872-04:00 stderr F level=info ts=2026-06-11T01:34:19.029928667Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:35:19.110209923-04:00 stderr F level=info ts=2026-06-11T01:35:19.029877769Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:36:19.111002317-04:00 stderr F level=info ts=2026-06-11T01:36:19.029509066Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:37:19.110992878-04:00 stderr F level=info ts=2026-06-11T01:37:19.029504797Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:37:49.110638232-04:00 stderr F level=info ts=2026-06-11T01:37:49.057800898Z caller=checkpoint.go:498 msg="atomic checkpoint finished" old=/tmp/loki/wal/checkpoint.000012.tmp new=/tmp/loki/wal/checkpoint.000012
2026-06-10T21:37:49.110638232-04:00 stderr F level=info ts=2026-06-11T01:37:49.05852602Z caller=checkpoint.go:569 msg="checkpoint done" time=4m30.010384128s
2026-06-10T21:38:19.110700580-04:00 stderr F level=info ts=2026-06-11T01:38:19.029461767Z caller=table_manager.go:228 index-store=tsdb-2020-10-24 msg="syncing tables"
2026-06-10T21:38:19.110700580-04:00 stderr F ts=2026-06-11T01:38:19.029536129Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:38:19.110700580-04:00 stderr F ts=2026-06-11T01:38:19.02956724Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=19.94µs
2026-06-10T21:38:19.110700580-04:00 stderr F ts=2026-06-11T01:38:19.029596921Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:38:19.110700580-04:00 stderr F ts=2026-06-11T01:38:19.029610152Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=6.621µs
2026-06-10T21:38:19.110700580-04:00 stderr F level=info ts=2026-06-11T01:38:19.029626932Z caller=table_manager.go:271 index-store=tsdb-2020-10-24 msg="query readiness setup completed" duration=1.67µs distinct_users_len=0 distinct_users=
2026-06-10T21:38:19.110700580-04:00 stderr F level=info ts=2026-06-11T01:38:19.029660763Z caller=table_manager.go:136 index-store=tsdb-2020-10-24 msg="uploading tables"
2026-06-10T21:38:19.110700580-04:00 stderr F level=info ts=2026-06-11T01:38:19.047307413Z caller=checkpoint.go:611 msg="starting checkpoint"
2026-06-10T21:38:19.110700580-04:00 stderr F level=info ts=2026-06-11T01:38:19.047508869Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/tmp/loki/wal/checkpoint.000013
2026-06-10T21:38:24.410453769-04:00 stderr F ts=2026-06-11T01:38:24.341164721Z caller=spanlogger.go:86 level=info msg="building table names cache"
2026-06-10T21:38:24.410453769-04:00 stderr F ts=2026-06-11T01:38:24.341268635Z caller=spanlogger.go:86 level=info msg="table names cache built" duration=26.722µs