diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go index eb82085d6f79..20d9dadb6c8d 100644 --- a/pkg/pattern/drain/drain.go +++ b/pkg/pattern/drain/drain.go @@ -103,12 +103,46 @@ type Node struct { } func DefaultConfig() *Config { + // TODO(kolesnikovae): + // + // This is crucial for Drain to ensure that the first LogClusterDepth tokens + // are constant (see https://jiemingzhu.github.io/pub/pjhe_icws2017.pdf). + // We should remove any variables such as timestamps, IDs, IPs, counters, etc. + // from these tokens. + // + // Moreover, Drain is not designed for structured logs. Therefore, we should + // handle logfmt (and, probably, JSON) logs in a special way: + // + // The parse tree should have a fixed length, and the depth should be + // determined by the number of fields in the logfmt message. + // A parsing tree should be maintained for each unique field set. return &Config{ + // At training, if at the depth of LogClusterDepth there is a cluster with + // similarity coefficient greater that SimTh, then the log message is added + // to that cluster. Otherwise, a new cluster is created. + // + // LogClusterDepth should be equal to the number of constant tokens from + // the beginning of the message that likely determine the message contents. + // + // > In this step, Drain traverses from a 1-st layer node, which + // > is searched in step 2, to a leaf node. This step is based on + // > the assumption that tokens in the beginning positions of a log + // > message are more likely to be constants. Specifically, Drain + // > selects the next internal node by the tokens in the beginning + // > positions of the log message LogClusterDepth: 8, - SimTh: 0.3, - MaxChildren: 100, - ParamString: "<_>", - MaxClusters: 0, + // SimTh is basically a ratio of matching/total in the cluster. + // Cluster tokens: "foo <*> bar fred" + // Log line: "foo bar baz qux" + // * * * x + // Similarity of these sequences is 0.75 (the distance) + // Both SimTh and MaxClusterDepth impact branching factor: the greater + // MaxClusterDepth and SimTh, the less the chance that there will be + // "similar" clusters, but the greater the footprint. + SimTh: 0.3, + MaxChildren: 100, + ParamString: `<_>`, + MaxClusters: 300, } } diff --git a/pkg/pattern/drain/drain_test.go b/pkg/pattern/drain/drain_test.go index 74b1e76ede12..beb09742af93 100644 --- a/pkg/pattern/drain/drain_test.go +++ b/pkg/pattern/drain/drain_test.go @@ -1 +1,118 @@ package drain + +import ( + "bufio" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDrain_TrainExtractsPatterns(t *testing.T) { + tests := []struct { + name string + drain *Drain + inputFile string + patterns []string + }{ + { + // High variation leads to many patterns including some that are too generic (many tokens matched) and some that are too specific (too few matchers) + name: `Generate patterns on high variation logfmt logs`, + drain: New(DefaultConfig()), + inputFile: `testdata/agent-logfmt.txt`, + patterns: []string{ + `ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}"`, + `<_> <_> level=info component=logs logs_config=default <_> target" <_> <_> <_> <_> <_> <_>`, + `<_> caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" <_>`, + `<_> caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" <_> reason=null`, + `<_> caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" <_>`, + `<_> caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened <_>`, + `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long"`, + `<_> caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" <_> op=CREATE`, + `<_> caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos <_> on line 1: unexpected '\"'"`, + `<_> <_> level=info component=logs logs_config=default <_> <_> <_> <_> <_>`, + `<_> caller=log.go:168 component=logs logs_config=default level=info <_> <_> <_> <_> <_>`, + `<_> caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" <_>`, + `<_> <_> level=info component=logs logs_config=default <_> target" <_> conprof=\"true\", <_> <_> job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", <_> plan=\"free\", <_> <_> <_> <_> <_>`, + `<_> level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines <_> <_>`, + `2024-04-16 15:10:42.555 ts=2024-04-16T15:10:42.555230437Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.545339ms`, + }, + }, + { + // Lower variation leads to fewer patterns including some with limited value (single lines, no matchers) + name: `Generate patterns on low variation logfmt logs`, + drain: New(DefaultConfig()), + inputFile: `testdata/ingester-logfmt.txt`, + patterns: []string{ + `<_> caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory`, + `ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms"`, + `<_> caller=http.go:194 level=debug <_> <_> msg="POST /ingester.v1.IngesterService/Push (200) <_>`, // A perfect log line: Abstracted the variable part but kept the constants. + }, + }, + { + // Lower variation logs in json leads to a high number of patterns with very few matchers + name: `Generate patterns on json formatted logs`, + drain: New(DefaultConfig()), + inputFile: `testdata/drone-json.txt`, + patterns: []string{ + `<_> capacity <_>`, + `<_> capacity changes <_>`, + `{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"}`, + `{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"}`, + `{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"}`, + `{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"}`, + `{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"}`, + `{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"}`, + `{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"}`, + `{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"}`, + `{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"}`, + `{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"}`, + `{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"}`, + `{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"}`, + `{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"}`, + `{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"}`, + `{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"}`, + `{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"}`, + `{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"}`, + `{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"}`, + `{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"}`, + `{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"}`, + `{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"}`, + `{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"}`, + `{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"}`, + `{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"}`, + `{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"}`, + `{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"}`, + `{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"}`, + `{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"}`, + `{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"}`, + `{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"}`, + `<_> server <_>`, + `<_> unfinished <_>`, + `<_> <_> (flow; linux; helm)"}`, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + file, err := os.Open(tt.inputFile) + require.NoError(t, err) + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + tt.drain.Train(line, 0) + } + + var output []string + clusters := tt.drain.Clusters() + for _, cluster := range clusters { + output = append(output, cluster.String()) + } + + require.Equal(t, tt.patterns, output) + }) + } +} diff --git a/pkg/pattern/drain/testdata/agent-logfmt.txt b/pkg/pattern/drain/testdata/agent-logfmt.txt new file mode 100644 index 000000000000..096c283193e3 --- /dev/null +++ b/pkg/pattern/drain/testdata/agent-logfmt.txt @@ -0,0 +1,1000 @@ +ts=2024-04-16T15:10:44.103611953Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log op=CREATE +ts=2024-04-16T15:10:44.102912583Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151044 op=CREATE +ts=2024-04-16T15:10:44.097147346Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151002.gz op=CREATE +ts=2024-04-16T15:10:44.053538158Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:44.040467356Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:44.040259872Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:44.020663216Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log" +ts=2024-04-16T15:10:44.020589942Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log ..." +ts=2024-04-16T15:10:43.995581296Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins/0.log +ts=2024-04-16T15:10:43.995566217Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.995488725Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-bccd55b-fn2h4_0d138adf-7938-4475-8248-05172a562c00/hg-plugins +ts=2024-04-16T15:10:43.994892796Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins/0.log +ts=2024-04-16T15:10:43.994870631Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.994764966Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-867b55884f-nbk4p_18703b31-d5eb-4108-9456-4c937645e5bd/hg-plugins +ts=2024-04-16T15:10:43.994701713Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins/0.log +ts=2024-04-16T15:10:43.994682386Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.994616509Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins/0.log +ts=2024-04-16T15:10:43.994581626Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.994431069Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-6b57fc5bd5-7z5dk_04c29f31-2bfd-463a-8e98-278d15895df8/hg-plugins +ts=2024-04-16T15:10:43.994354906Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-67fdffbbcd-wrs8v_a1ec727b-6624-42f0-8cc1-b3a07c1bda71/hg-plugins +ts=2024-04-16T15:10:43.991326425Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-7h75h_3013c1d4-6a76-4548-8eab-d4b71ef0bccd/grafana-ruler/0.log.20240416-150803.tmp op=CREATE +ts=2024-04-16T15:10:43.987816559Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log op=CREATE +ts=2024-04-16T15:10:43.987450089Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.985283077Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-150642.gz op=CREATE +ts=2024-04-16T15:10:43.963516043Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log" +ts=2024-04-16T15:10:43.961050802Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log ..." +ts=2024-04-16T15:10:43.960201305Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.960173523Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins/0.log +ts=2024-04-16T15:10:43.960099063Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-ffdfcb6f6-vtm7j_1b0ba139-a4c5-4534-98bd-b2c66e4205cb/hg-plugins +ts=2024-04-16T15:10:43.956496124Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.956459518Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins/0.log +ts=2024-04-16T15:10:43.956333382Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-558c5cff54-58lhv_14f770e0-e9d2-4957-ad3c-51bf7bab9c38/hg-plugins +ts=2024-04-16T15:10:43.94285492Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.942820378Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins/0.log +ts=2024-04-16T15:10:43.942720436Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-6f59c6b77d-b4z6m_39598760-48b2-4b37-8693-e511348d3910/hg-plugins +ts=2024-04-16T15:10:43.93643872Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.936406569Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins/0.log +ts=2024-04-16T15:10:43.936280507Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-747b4b8bb6-46kwt_852b076d-c773-4e47-bc55-d7eeae8a5837/hg-plugins +ts=2024-04-16T15:10:43.93512145Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log" +ts=2024-04-16T15:10:43.935029359Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log ..." +ts=2024-04-16T15:10:43.917279646Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/billing_billing-forwarder-5bf8799d98-jx47p_93a99fc6-f43b-44fd-ae92-f1d8e79639ba/billing-forwarder/0.log.20240416-151002.tmp op=CREATE +ts=2024-04-16T15:10:43.912837401Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.912678958Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hgrun/0.log ..." +ts=2024-04-16T15:10:43.913001917Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.910710328Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log op=CREATE +ts=2024-04-16T15:10:43.910230202Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.90589892Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151033.gz op=CREATE +ts=2024-04-16T15:10:43.902671587Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log op=CREATE +ts=2024-04-16T15:10:43.902293197Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.901374194Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-121049.gz op=CREATE +ts=2024-04-16T15:10:43.886121557Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.885839882Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.882665637Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log op=CREATE +ts=2024-04-16T15:10:43.8818563Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.878087537Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151013.gz op=CREATE +ts=2024-04-16T15:10:43.870249341Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log +ts=2024-04-16T15:10:43.870231421Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log +ts=2024-04-16T15:10:43.87016127Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log +ts=2024-04-16T15:10:43.87014674Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log +ts=2024-04-16T15:10:43.870126919Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.870016638Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}" +ts=2024-04-16T15:10:43.869947507Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log +ts=2024-04-16T15:10:43.869933646Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log +ts=2024-04-16T15:10:43.869919827Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log +ts=2024-04-16T15:10:43.869901596Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log +ts=2024-04-16T15:10:43.869885086Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/0.log reason=null +ts=2024-04-16T15:10:43.869833185Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}" +ts=2024-04-16T15:10:43.869772754Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log +ts=2024-04-16T15:10:43.869758435Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log +ts=2024-04-16T15:10:43.869740834Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log +ts=2024-04-16T15:10:43.869727154Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log +ts=2024-04-16T15:10:43.869713063Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.869672113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}" +ts=2024-04-16T15:10:43.869543361Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.869526821Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.869511381Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.869496801Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.869477971Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-666bd48cf9-7zrtv_37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.869370539Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*37ae8d4e-1a76-40f2-be88-2251a3528a0b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i2222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"someorg\", plan=\"free\", pod=\"someorg-grafana-666bd48cf9-7zrtv\", pod_template_hash=\"666bd48cf9\", resource_version=\"167212086\", slug=\"someorg\", stackId=\"444444\"}" +ts=2024-04-16T15:10:43.866539497Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-lbf6f_1b71b005-4a57-4f52-acda-734f074e93c9/distributor/0.log.20240416-150602.tmp op=CREATE +ts=2024-04-16T15:10:43.862521938Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-24_graphite-write-proxy-56c899b9bd-sv5wk_88dab27e-6880-4eab-84ca-23181ddf59f0/graphite-write-proxy/0.log.20240416-150933.tmp op=CREATE +ts=2024-04-16T15:10:43.854830425Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.84883762Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.845881472Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.845202349Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-w87sj_04264a7e-491d-4c22-b109-c97f9b227eb0/profiler/0.log.20240416-121049.tmp op=CREATE +ts=2024-04-16T15:10:43.834940494Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/*.log +ts=2024-04-16T15:10:43.834919864Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log +ts=2024-04-16T15:10:43.834891984Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log +ts=2024-04-16T15:10:43.834870753Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log +ts=2024-04-16T15:10:43.834850373Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log reason=null +ts=2024-04-16T15:10:43.834799833Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log +ts=2024-04-16T15:10:43.834752462Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}" +ts=2024-04-16T15:10:43.834460948Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/*.log +ts=2024-04-16T15:10:43.833997181Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log +ts=2024-04-16T15:10:43.833967761Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log +ts=2024-04-16T15:10:43.83395078Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log +ts=2024-04-16T15:10:43.83393401Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.8338997Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log +ts=2024-04-16T15:10:43.8338763Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}" +ts=2024-04-16T15:10:43.833819339Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/*.log +ts=2024-04-16T15:10:43.833794999Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log +ts=2024-04-16T15:10:43.838457484Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.837161525Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.833775169Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log +ts=2024-04-16T15:10:43.833756968Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log +ts=2024-04-16T15:10:43.833697477Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.833662116Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log +ts=2024-04-16T15:10:43.833629856Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}" +ts=2024-04-16T15:10:43.833457274Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.833433624Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.833400573Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.833382792Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.833364072Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.833319402Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.833226031Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"222222\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org3\", plan=\"free\", pod=\"org3-grafana-7fd6786f4b-242cb\", pod_template_hash=\"7fd6786f4b\", resource_version=\"167282051\", slug=\"org3\", stackId=\"1111111\"}" +ts=2024-04-16T15:10:43.820104119Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log" +ts=2024-04-16T15:10:43.819972587Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log ..." +ts=2024-04-16T15:10:43.825845873Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.82429901Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.812714489Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-7cb758bfd6-hcq2d_c3259533-8d14-4b65-a6d0-c3880de4feba/grafana-ruler/0.log.20240416-150642.tmp op=CREATE +ts=2024-04-16T15:10:43.811775736Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.789499114Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.789387503Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d89d8b5fd-lqw92_e60bfc3b-9967-4a9d-8370-ace8d046123b/grafana/0.log ..." +ts=2024-04-16T15:10:43.763317681Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.73769616Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-18_e3cf5e0a-b9ba-4be0-b3b0-6157f3cca0f7/ingester/0.log.20240416-150613.tmp op=CREATE +ts=2024-04-16T15:10:43.717654564Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-012_query-frontend-7f5bf94766-r6nx7_cf30b042-cf38-4562-a431-26e7e37665c2/query-frontend/4.log.20240416-151033.tmp op=CREATE +ts=2024-04-16T15:10:43.715625202Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.710462108Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.706609368Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.699754134Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ruler-c957f4f69-stmzv_cfa34492-c676-42ce-b71c-5793aa5d1488/ruler/0.log.20240416-151033.tmp op=CREATE +ts=2024-04-16T15:10:43.689150897Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.671351341Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.66739874Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log" +ts=2024-04-16T15:10:43.667327798Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log ..." +ts=2024-04-16T15:10:43.665742041Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log" +ts=2024-04-16T15:10:43.665664607Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log ..." +ts=2024-04-16T15:10:43.624354483Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log op=CREATE +ts=2024-04-16T15:10:43.618647073Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.614330743Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-85ddb8cd7b-qdw4m_700c6bab-d080-42d2-b8bf-7d6b3180395a/grafana/0.log.20240416-150932.tmp op=CREATE +ts=2024-04-16T15:10:43.613342016Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.613114354Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/grafana/0.log ..." +ts=2024-04-16T15:10:43.612632997Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151011.gz op=CREATE +ts=2024-04-16T15:10:43.608713774Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log op=CREATE +ts=2024-04-16T15:10:43.607662962Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.604544245Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151023.gz op=CREATE +ts=2024-04-16T15:10:43.593678478Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.59255473Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.592407837Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hgrun/0.log ..." +ts=2024-04-16T15:10:43.587217309Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.5862021Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.585137415Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.584968803Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.585003039Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.580159324Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.579992546Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.569307522Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.56817841Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.56492517Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-gateway-78cb874c8f-2pmlc_26b57c76-eabc-4d29-9305-3f034c24e147/istio-proxy/0.log.20240416-151013.tmp op=CREATE +ts=2024-04-16T15:10:43.563399768Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.563325868Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.555169973Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.553816164Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log op=CREATE +ts=2024-04-16T15:10:43.553398207Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.552323479Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-150452.gz op=CREATE +ts=2024-04-16T15:10:43.55175804Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.55169237Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hgrun/0.log ..." +ts=2024-04-16T15:10:43.551819984Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log +ts=2024-04-16T15:10:43.551782223Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log +ts=2024-04-16T15:10:43.551748001Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log +ts=2024-04-16T15:10:43.551719538Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log +ts=2024-04-16T15:10:43.551673641Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log reason=null +ts=2024-04-16T15:10:43.551621071Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/grafana-com_marketplaces-api-f67ff7567-gqrvb_35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/0.log +ts=2024-04-16T15:10:43.551543875Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*35649bfd-52ff-4281-9294-5f65fd5a89fc/marketplaces-api/*.log:{container=\"marketplaces-api\", job=\"grafana-com/marketplaces-api\", name=\"marketplaces-api\", namespace=\"grafana-com\", pod=\"marketplaces-api-f67ff7567-gqrvb\", pod_template_hash=\"f67ff7567\"}" +ts=2024-04-16T15:10:43.551598289Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.539801042Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.53959404Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hgrun/0.log ..." +ts=2024-04-16T15:10:43.532863601Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.53265924Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.524399551Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/calico-node/0.log to appear..." +ts=2024-04-16T15:10:43.524115935Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/calico-node/0.log ..." +ts=2024-04-16T15:10:43.523998271Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/install-cni/0.log to appear..." +ts=2024-04-16T15:10:43.523842724Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-7zxvh_a1ad8906-41f3-46de-9fb7-64656624556b/install-cni/0.log ..." +ts=2024-04-16T15:10:43.522251853Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.522135262Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/grafana/0.log ..." +ts=2024-04-16T15:10:43.513279228Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/install-cni/0.log to appear..." +ts=2024-04-16T15:10:43.513186063Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/install-cni/0.log ..." +ts=2024-04-16T15:10:43.508349919Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/*.log +ts=2024-04-16T15:10:43.508329009Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log +ts=2024-04-16T15:10:43.508309139Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log +ts=2024-04-16T15:10:43.508287738Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log +ts=2024-04-16T15:10:43.508267968Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.508237718Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log +ts=2024-04-16T15:10:43.508173806Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}" +ts=2024-04-16T15:10:43.507960012Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.507831693Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/grafana/*.log +ts=2024-04-16T15:10:43.507797611Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log +ts=2024-04-16T15:10:43.507781221Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log +ts=2024-04-16T15:10:43.507765151Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log +ts=2024-04-16T15:10:43.507730981Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log reason=null +ts=2024-04-16T15:10:43.5076889Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log +ts=2024-04-16T15:10:43.50762108Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}" +ts=2024-04-16T15:10:43.507460368Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.507397087Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.507344086Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.507308256Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.507283825Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.507259015Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.507208275Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}" +ts=2024-04-16T15:10:43.507100714Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/*.log +ts=2024-04-16T15:10:43.507027892Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log +ts=2024-04-16T15:10:43.506996362Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log +ts=2024-04-16T15:10:43.506977991Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log +ts=2024-04-16T15:10:43.506953691Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.506923011Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log +ts=2024-04-16T15:10:43.50681944Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i3333\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org4\", plan=\"free\", pod=\"org4-grafana-b4f87fcc5-fflgn\", pod_template_hash=\"b4f87fcc5\", resource_version=\"167289888\", slug=\"org4\", stackId=\"333333\"}" +ts=2024-04-16T15:10:43.503581356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log op=CREATE +ts=2024-04-16T15:10:43.50308684Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.499582153Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-145410.gz op=CREATE +ts=2024-04-16T15:10:43.491065781Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.491039691Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins/0.log +ts=2024-04-16T15:10:43.490892989Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hg-plugins +ts=2024-04-16T15:10:43.485641743Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun/0.log +ts=2024-04-16T15:10:43.485611252Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.485462621Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/hgrun +ts=2024-04-16T15:10:43.485235288Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana/0.log +ts=2024-04-16T15:10:43.485241618Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.484833293Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-5f848b765-hmj42_31c37ccc-044a-4bed-a85f-358870a3a02c/grafana +ts=2024-04-16T15:10:43.485316721Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log" +ts=2024-04-16T15:10:43.485223151Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log ..." +ts=2024-04-16T15:10:43.483728092Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.475061435Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.471990067Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.47184237Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.471539735Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.459291911Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.454815097Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.442284255Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-wsddd_b19cb3ac-7212-48e8-a74c-573d5732f2a9/grafana/0.log.20240416-151023.tmp op=CREATE +ts=2024-04-16T15:10:43.440096821Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.424305955Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.421737727Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log op=CREATE +ts=2024-04-16T15:10:43.420163091Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.416217316Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/mimir-dedicated-48_ingester-zone-b-72_1f08c7df-e68c-4101-afba-3cb84acb34f0/ingester/0.log.20240416-150452.tmp op=CREATE +ts=2024-04-16T15:10:43.413967547Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151032.gz op=CREATE +ts=2024-04-16T15:10:43.409009601Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log" +ts=2024-04-16T15:10:43.408920067Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log ..." +ts=2024-04-16T15:10:43.401825217Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/insight-logs_promtail-insight-logs-pxhrc_c86ff41c-f07b-4d03-aef7-05d9e243cef6/promtail/0.log.20240416-145410.tmp op=CREATE +ts=2024-04-16T15:10:43.382088048Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.382307404Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.381990902Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.38181857Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.381333325Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.378767789Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.378415347Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/grafana/0.log ..." +ts=2024-04-16T15:10:43.375853671Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.375689458Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.370564138Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-45_bd1d50cc-1edf-4070-b4e9-aca09955fbc2/compactor/0.log.20240416-150722.tmp op=CREATE +ts=2024-04-16T15:10:43.361010102Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log" +ts=2024-04-16T15:10:43.368284509Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log ..." +ts=2024-04-16T15:10:43.366399734Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.366192752Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d8c765c5-hsdpm_6cbab356-d034-4ff8-9879-077d7f363956/hgrun/0.log ..." +ts=2024-04-16T15:10:43.349652417Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.349452285Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.344818625Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.344791104Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.344771214Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/grafana/0.log ..." +ts=2024-04-16T15:10:43.344715123Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b9fc57d4d-sctgm_af150893-4992-4f5e-aa4d-5f178e76f33a/hgrun/0.log ..." +ts=2024-04-16T15:10:43.339861011Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log op=CREATE +ts=2024-04-16T15:10:43.339463329Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.338895596Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.338376505Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-150933.gz op=CREATE +ts=2024-04-16T15:10:43.33378223Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.333585597Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-66bd77fc5d-w6frp_e21695f7-52de-495e-8d78-741519657226/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.332851752Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log" +ts=2024-04-16T15:10:43.332775064Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log ..." +ts=2024-04-16T15:10:43.328013161Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.327593586Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.323077753Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.322751558Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/grafana/0.log ..." +ts=2024-04-16T15:10:43.319886478Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.320037046Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.320023386Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.320011954Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.319989552Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.319945602Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.319933664Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/grafana/0.log ..." +ts=2024-04-16T15:10:43.314580717Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.314408635Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-58687d789-49wfb_269bf9a8-7811-4259-b1fc-ff487ffc9a9a/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.314451491Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.314350274Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-8c9f94cf-4npds_48df0ae1-00a4-4c5f-9d89-8c55b50c08f4/hgrun/0.log ..." +ts=2024-04-16T15:10:43.308822084Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.306240475Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log op=CREATE +ts=2024-04-16T15:10:43.305755943Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.304574308Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.302851526Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.302759616Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-6766cd4c64-v7ccz_c8ec4c12-78c6-4f20-b382-8ae7e9f0f4af/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.302148378Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-150742.gz op=CREATE +ts=2024-04-16T15:10:43.297925853Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.297912585Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.297901068Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.297890333Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.297880399Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.297866745Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.297841906Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}" +ts=2024-04-16T15:10:43.297823221Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/grafana/*.log +ts=2024-04-16T15:10:43.297808137Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log +ts=2024-04-16T15:10:43.297793002Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log +ts=2024-04-16T15:10:43.29777652Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log +ts=2024-04-16T15:10:43.297763874Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log reason=null +ts=2024-04-16T15:10:43.297751195Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/grafana/0.log +ts=2024-04-16T15:10:43.297733011Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}" +ts=2024-04-16T15:10:43.297678961Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/*.log +ts=2024-04-16T15:10:43.29766299Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log +ts=2024-04-16T15:10:43.297646746Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log +ts=2024-04-16T15:10:43.297636834Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log +ts=2024-04-16T15:10:43.297624194Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.297610295Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/0.log +ts=2024-04-16T15:10:43.297594201Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}" +ts=2024-04-16T15:10:43.297572495Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/*.log +ts=2024-04-16T15:10:43.297555846Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log +ts=2024-04-16T15:10:43.29753931Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log +ts=2024-04-16T15:10:43.297520943Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log +ts=2024-04-16T15:10:43.29750834Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.297478804Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-ddbf649cc-zgtf6_9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/0.log +ts=2024-04-16T15:10:43.297420444Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*9afcdf84-163b-402e-bdd2-cfb711593385/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i4444\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org5\", plan=\"free\", pod=\"org5-grafana-ddbf649cc-zgtf6\", pod_template_hash=\"ddbf649cc\", resource_version=\"95783554\", slug=\"org5\", stackId=\"555555\"}" +ts=2024-04-16T15:10:43.296157354Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.296139225Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}" +ts=2024-04-16T15:10:43.29611646Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.296116023Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/*.log +ts=2024-04-16T15:10:43.296101716Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}" +ts=2024-04-16T15:10:43.296098008Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.296083728Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.2960726Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.296071337Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/*.log +ts=2024-04-16T15:10:43.296059925Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}" +ts=2024-04-16T15:10:43.296036364Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.296037858Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/*.log +ts=2024-04-16T15:10:43.296011595Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security +ts=2024-04-16T15:10:43.296005269Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i5555\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org6\", plan=\"free\", pod=\"org6-grafana-fdfdc64bb-srrx6\", pod_template_hash=\"fdfdc64bb\", resource_version=\"95745089\", slug=\"org6\", stackId=\"666666\"}" +ts=2024-04-16T15:10:43.295945337Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log +ts=2024-04-16T15:10:43.295931888Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log +ts=2024-04-16T15:10:43.295920924Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log +ts=2024-04-16T15:10:43.295903261Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.295876551Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log +ts=2024-04-16T15:10:43.295857948Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins +ts=2024-04-16T15:10:43.295670559Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7c67b65cdd-mk82w_496ab238-1eed-4b76-8345-de4537cb7c0e/grafana/0.log.20240416-151011.tmp op=CREATE +ts=2024-04-16T15:10:43.295672083Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log +ts=2024-04-16T15:10:43.295659368Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log +ts=2024-04-16T15:10:43.295649791Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log +ts=2024-04-16T15:10:43.295626379Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.295589808Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log op=CREATE +ts=2024-04-16T15:10:43.295587723Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log +ts=2024-04-16T15:10:43.295545284Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun +ts=2024-04-16T15:10:43.295472158Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.295436643Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log +ts=2024-04-16T15:10:43.295422636Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log +ts=2024-04-16T15:10:43.295411202Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log +ts=2024-04-16T15:10:43.295395484Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log reason=null +ts=2024-04-16T15:10:43.295370159Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hg-plugins/0.log +ts=2024-04-16T15:10:43.295364051Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log +ts=2024-04-16T15:10:43.295336559Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana +ts=2024-04-16T15:10:43.295169248Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.294932482Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/hgrun/0.log +ts=2024-04-16T15:10:43.294837816Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-fdfdc64bb-srrx6_0ecafb81-c168-4bc4-99e3-e8b2315a09b8/grafana/0.log +ts=2024-04-16T15:10:43.294695226Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod-026_query-frontend-7fc658ff96-xhbns_91d07b6a-76db-4267-a421-9616df7ab198/query-frontend/0.log" +ts=2024-04-16T15:10:43.294614823Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod-026_query-frontend-7fc658ff96-xhbns_91d07b6a-76db-4267-a421-9616df7ab198/query-frontend/0.log ..." +ts=2024-04-16T15:10:43.291684718Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-135703.gz op=CREATE +ts=2024-04-16T15:10:43.291663946Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log" +ts=2024-04-16T15:10:43.291549231Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log ..." +ts=2024-04-16T15:10:43.285237795Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log" +ts=2024-04-16T15:10:43.28515592Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log ..." +ts=2024-04-16T15:10:43.267196378Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log op=CREATE +ts=2024-04-16T15:10:43.266859931Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.256583646Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/*.log +ts=2024-04-16T15:10:43.256553405Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.256428125Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.256408755Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log +ts=2024-04-16T15:10:43.256365475Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:43.256261263Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}" +ts=2024-04-16T15:10:43.256183643Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/*.log +ts=2024-04-16T15:10:43.256166953Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log +ts=2024-04-16T15:10:43.256150033Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log +ts=2024-04-16T15:10:43.256132803Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log +ts=2024-04-16T15:10:43.256115503Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/0.log reason=null +ts=2024-04-16T15:10:43.256072092Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}" +ts=2024-04-16T15:10:43.256024602Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/*.log +ts=2024-04-16T15:10:43.256006782Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log +ts=2024-04-16T15:10:43.255989251Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log +ts=2024-04-16T15:10:43.255974991Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log +ts=2024-04-16T15:10:43.255960211Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:43.255933401Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log +ts=2024-04-16T15:10:43.255907671Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}" +ts=2024-04-16T15:10:43.25582646Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/grafana/*.log +ts=2024-04-16T15:10:43.25580358Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log +ts=2024-04-16T15:10:43.25578448Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log +ts=2024-04-16T15:10:43.25576612Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log +ts=2024-04-16T15:10:43.25574394Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/grafana/0.log reason=null +ts=2024-04-16T15:10:43.255643719Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*64872ae4-62eb-4757-b148-72bab4a9e880/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i6666\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org7\", plan=\"free\", pod=\"org7-grafana-647dc5b44f-pmz8j\", pod_template_hash=\"647dc5b44f\", resource_version=\"167297262\", slug=\"org7\", stackId=\"777777\"}" +ts=2024-04-16T15:10:43.263042477Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-150933.gz op=CREATE +ts=2024-04-16T15:10:43.259110088Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.257819383Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.244818982Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.239151767Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.239080083Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main/0.log +ts=2024-04-16T15:10:43.238913661Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/loki-cd_loki-cd-loki-create-deployment-tuesday-prod-1713254400-wait-for-github-pr-862956382_20556f0a-8a98-4361-bb91-baa2ffb08c41/main +ts=2024-04-16T15:10:43.234330247Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.234253033Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hgrun/0.log ..." +ts=2024-04-16T15:10:43.231425217Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.231145214Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hgrun/0.log ..." +ts=2024-04-16T15:10:43.220000621Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log op=CREATE +ts=2024-04-16T15:10:43.21939049Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.219314914Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.215848733Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.215641301Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-150450.gz op=CREATE +ts=2024-04-16T15:10:43.213328712Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.212676387Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.212298621Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.209036752Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/alertmanager_alertmanager-3_a068a2ab-d658-44d2-9b78-d9d7eb47e7cf/alertmanager/0.log.20240416-150933.tmp op=CREATE +ts=2024-04-16T15:10:43.207714985Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log" +ts=2024-04-16T15:10:43.207654185Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log ..." +ts=2024-04-16T15:10:43.207288069Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log" +ts=2024-04-16T15:10:43.207202372Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log ..." +ts=2024-04-16T15:10:43.206926893Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log" +ts=2024-04-16T15:10:43.206845577Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log ..." +ts=2024-04-16T15:10:43.205820894Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.205713872Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.202126267Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.196717804Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/0.log +ts=2024-04-16T15:10:43.196713431Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.19648143Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/k6-cloud_testcoordinator-job-2665838-9g8ds_19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator +ts=2024-04-16T15:10:43.196210056Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_aggregator-40-7d876c497f-qlb2t_f99739bd-bd1f-4605-aaf0-61ef442f4610/aggregator/0.log.20240416-150742.tmp op=CREATE +ts=2024-04-16T15:10:43.192290389Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*19a1cce8-5f04-46e0-a124-292b0dd9b343/testcoordinator/*.log:{batch_kubernetes_io_controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", batch_kubernetes_io_job_name=\"testcoordinator-job-2665838\", container=\"testcoordinator\", controller_uid=\"25ec5edf-f78e-468b-b6f3-3b9685f0cc8f\", job=\"k6-cloud/testcoordinator\", job_name=\"testcoordinator-job-2665838\", name=\"testcoordinator\", namespace=\"k6-cloud\", pod=\"testcoordinator-job-2665838-9g8ds\"}" +ts=2024-04-16T15:10:43.177328136Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.177232412Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-dd8bf9cd-s8k4m_02f6e4f5-aad2-445f-a04c-f666f0d54ee4/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.187183919Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/install-cni/0.log to appear..." +ts=2024-04-16T15:10:43.187062724Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/install-cni/0.log ..." +ts=2024-04-16T15:10:43.18059824Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.174053022Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.172105378Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.170592658Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log" +ts=2024-04-16T15:10:43.170496061Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log ..." +ts=2024-04-16T15:10:43.170424393Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log op=CREATE +ts=2024-04-16T15:10:43.170071302Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.168507042Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log op=CREATE +ts=2024-04-16T15:10:43.167505496Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.16743356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-150602.gz op=CREATE +ts=2024-04-16T15:10:43.16346356Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-150902.gz op=CREATE +ts=2024-04-16T15:10:43.162766766Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.162659451Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.154757178Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.154349425Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.144637923Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.144559282Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.136197313Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.136128662Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.135428948Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:43.13241598Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.1323331Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-64dc5cbf86-nvgx5_b5380684-df51-45a4-bacb-ca0cfdf7a05f/grafana/0.log ..." +ts=2024-04-16T15:10:43.12931936Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-7d484c79c6-hzqs8_daf4dc8d-7371-48e5-8a18-cda6289c7f2a/grafana/0.log.20240416-151032.tmp op=CREATE +ts=2024-04-16T15:10:43.127063794Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.127036834Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun/0.log +ts=2024-04-16T15:10:43.126942786Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins/0.log +ts=2024-04-16T15:10:43.126913616Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hgrun +ts=2024-04-16T15:10:43.126921946Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.126750144Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-58bc4b54f9-v2hs8_64317ae0-8f2c-4861-8629-4363b46f550d/hg-plugins +ts=2024-04-16T15:10:43.126157093Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.125022226Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/pyroscope-ebpf_profiler-jt8kd_f4785188-1d36-4950-a6f5-328218c73e59/profiler/0.log.20240416-135703.tmp op=CREATE +ts=2024-04-16T15:10:43.120379101Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-wxfdz_47df6fbe-43af-4cf7-a77f-edc3de06197c/cortex-gw/0.log.20240416-150933.tmp op=CREATE +ts=2024-04-16T15:10:43.120021897Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log op=CREATE +ts=2024-04-16T15:10:43.119451076Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.116318645Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/calico-node/0.log to appear..." +ts=2024-04-16T15:10:43.115865529Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-jwmqj_894554c7-64d9-4221-9032-1d2966ce987f/calico-node/0.log ..." +ts=2024-04-16T15:10:43.115024147Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-150952.gz op=CREATE +ts=2024-04-16T15:10:43.108835306Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.108765136Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-647dc5b44f-pmz8j_64872ae4-62eb-4757-b148-72bab4a9e880/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.08743031Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log op=CREATE +ts=2024-04-16T15:10:43.087145122Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log" +ts=2024-04-16T15:10:43.086905187Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.086995387Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log ..." +ts=2024-04-16T15:10:43.083530447Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-150319.gz op=CREATE +ts=2024-04-16T15:10:43.082142726Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.081175944Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.081142393Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hgrun/0.log ..." +ts=2024-04-16T15:10:43.081118603Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.081069412Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.078778917Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/grafana/0.log to appear..." +ts=2024-04-16T15:10:43.078661804Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7f47dc8c57-7xkbj_94f05c74-2cbd-48ba-9109-b1966920123b/grafana/0.log ..." +ts=2024-04-16T15:10:43.065663262Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log op=CREATE +ts=2024-04-16T15:10:43.065151425Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:43.062372224Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-150942.gz op=CREATE +ts=2024-04-16T15:10:43.057467305Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*6e95d7c0-d863-461c-a6d1-68653e438e3e/kube-proxy/*.log:{component=\"kube-proxy\", container=\"kube-proxy\", job=\"kube-system/gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}" +ts=2024-04-16T15:10:43.053333798Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log" +ts=2024-04-16T15:10:43.053227989Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log ..." +ts=2024-04-16T15:10:43.047219694Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:43.047188126Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/0.log +ts=2024-04-16T15:10:43.047043424Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/kube-system_kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw_1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy +ts=2024-04-16T15:10:43.044285087Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*1954fd4ff7221e619e2d202bfb2c4ab9/kube-proxy/*.log:{component=\"kube-proxy\", container=\"kube-proxy\", job=\"kube-system/kube-proxy\", namespace=\"kube-system\", pod=\"kube-proxy-gke-ops-us-east-0-main-n2s32-1-1dd39c-32ae1dde-hmhw\", tier=\"node\"}" +ts=2024-04-16T15:10:43.020577028Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:43.020377695Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hg-plugins/0.log ..." +ts=2024-04-16T15:10:43.020680098Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/profiles-ops-001_fire-compactor-91_2be1ca60-4f1b-4f90-90bc-d235de54176b/compactor/0.log.20240416-150902.tmp op=CREATE +ts=2024-04-16T15:10:43.015377842Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log to appear..." +ts=2024-04-16T15:10:43.015197479Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-b4f87fcc5-fflgn_043372d7-9411-443f-ba7f-80988f77d8b3/hgrun/0.log ..." +ts=2024-04-16T15:10:43.008239745Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:43.008154235Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:43.010216068Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:43.003786403Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log op=CREATE +ts=2024-04-16T15:10:43.002376901Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151043 op=CREATE +ts=2024-04-16T15:10:42.999925258Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151012.gz op=CREATE +ts=2024-04-16T15:10:42.99578715Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log" +ts=2024-04-16T15:10:42.995688711Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log ..." +ts=2024-04-16T15:10:42.992974648Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/tempo-prod_ingester-65_02bae376-3900-4aaf-bc1c-717824388403/ingester/0.log.20240416-150952.tmp op=CREATE +ts=2024-04-16T15:10:42.988556779Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log" +ts=2024-04-16T15:10:42.988475443Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log ..." +ts=2024-04-16T15:10:42.988115995Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-d9d7957b8-w7pnf_6cfc70ca-26c6-4421-9483-ffe123fe3a6b/distributor/0.log.20240416-150450.tmp op=CREATE +ts=2024-04-16T15:10:42.98719306Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_distributor-76c559cb69-4v4k4_8a587464-bdea-4777-83a4-fbb81df911ec/distributor/0.log.20240416-150602.tmp op=CREATE +ts=2024-04-16T15:10:42.981209224Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log" +ts=2024-04-16T15:10:42.981121512Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log ..." +ts=2024-04-16T15:10:42.972859196Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.970735558Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.968185291Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.968148784Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.961558674Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log" +ts=2024-04-16T15:10:42.961469548Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log ..." +ts=2024-04-16T15:10:42.960769391Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.95311867Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.95302964Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/grafana/0.log ..." +ts=2024-04-16T15:10:42.959146686Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log op=CREATE +ts=2024-04-16T15:10:42.95884468Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.956246032Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/*.log +ts=2024-04-16T15:10:42.95623547Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log +ts=2024-04-16T15:10:42.956222919Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log +ts=2024-04-16T15:10:42.956211852Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log +ts=2024-04-16T15:10:42.956202082Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.956185492Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/0.log +ts=2024-04-16T15:10:42.956170942Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}" +ts=2024-04-16T15:10:42.956145757Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/*.log +ts=2024-04-16T15:10:42.956134619Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log +ts=2024-04-16T15:10:42.956120932Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log +ts=2024-04-16T15:10:42.956108765Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log +ts=2024-04-16T15:10:42.956098278Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log reason=null +ts=2024-04-16T15:10:42.956083968Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/0.log +ts=2024-04-16T15:10:42.956067383Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}" +ts=2024-04-16T15:10:42.95599928Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.95598394Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.955972198Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.955959785Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.955948953Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.955934083Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.955913962Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}" +ts=2024-04-16T15:10:42.955890015Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/*.log +ts=2024-04-16T15:10:42.955870893Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log +ts=2024-04-16T15:10:42.95585467Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log +ts=2024-04-16T15:10:42.955842788Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log +ts=2024-04-16T15:10:42.955828203Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.95580573Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-6c66686686-mqtcr_c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/0.log +ts=2024-04-16T15:10:42.95572753Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*c3c249a2-c8ff-40f4-a66d-9d746b39110b/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i7777\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org8\", plan=\"free\", pod=\"org8-grafana-6c66686686-mqtcr\", pod_template_hash=\"6c66686686\", resource_version=\"95723800\", slug=\"org8\", stackId=\"888888\"}" +ts=2024-04-16T15:10:42.95600412Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.9540311Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-150912.gz op=CREATE +ts=2024-04-16T15:10:42.943541527Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-4kkh4_0cc79d0a-6c37-414d-8a2a-0b91b0b3e6c2/query-frontend/4.log.20240416-150942.tmp op=CREATE +ts=2024-04-16T15:10:42.926967398Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.90779516Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/calico-node/0.log to appear..." +ts=2024-04-16T15:10:42.90688742Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/kube-system_calico-node-8dlq6_07f72b39-55dd-4e63-903d-0120cd9b3cda/calico-node/0.log ..." +ts=2024-04-16T15:10:42.904408857Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log op=CREATE +ts=2024-04-16T15:10:42.902260189Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-97c647648-whbkm_74aa1a7c-2b5b-4986-a95b-3cd30627d846/grafana/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.88299518Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.880612088Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/hosted-grafana_.something-grafana-d95d68499-sqqh8_9c491b2c-2dff-4a72-a3ef-c5719f5cf696/grafana/0.log.20240416-151012.tmp op=CREATE +ts=2024-04-16T15:10:42.878870475Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hgrun/0.log to appear..." +ts=2024-04-16T15:10:42.878760804Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hgrun/0.log ..." +ts=2024-04-16T15:10:42.87611453Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod-028_querier-5864bffd9f-pr8x4_f823f483-b830-4ebb-9bb3-fd6f35b49cfb/querier/1.log" +ts=2024-04-16T15:10:42.876032602Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod-028_querier-5864bffd9f-pr8x4_f823f483-b830-4ebb-9bb3-fd6f35b49cfb/querier/1.log ..." +ts=2024-04-16T15:10:42.857009279Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.85475629Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.85473729Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.85471274Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.85469692Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.854665989Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.853457993Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}" +ts=2024-04-16T15:10:42.853309562Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/*.log +ts=2024-04-16T15:10:42.853292992Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log +ts=2024-04-16T15:10:42.853277081Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log +ts=2024-04-16T15:10:42.853262091Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log +ts=2024-04-16T15:10:42.853248801Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.853229701Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log +ts=2024-04-16T15:10:42.853210051Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}" +ts=2024-04-16T15:10:42.85315126Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/*.log +ts=2024-04-16T15:10:42.853125369Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log +ts=2024-04-16T15:10:42.853110049Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log +ts=2024-04-16T15:10:42.853094689Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log +ts=2024-04-16T15:10:42.853080408Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log reason=null +ts=2024-04-16T15:10:42.853059658Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log +ts=2024-04-16T15:10:42.853040568Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}" +ts=2024-04-16T15:10:42.853013388Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/*.log +ts=2024-04-16T15:10:42.852994747Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log +ts=2024-04-16T15:10:42.852976767Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log +ts=2024-04-16T15:10:42.852962817Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log +ts=2024-04-16T15:10:42.852946987Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.852918397Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/0.log +ts=2024-04-16T15:10:42.852862236Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bd6a9960-6f03-46ee-b646-1456cff50d71/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i8888\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org9\", plan=\"free\", pod=\"org9-grafana-cccfb8fd6-cktl7\", pod_template_hash=\"cccfb8fd6\", resource_version=\"167320979\", slug=\"org9\", stackId=\"999999\"}" +ts=2024-04-16T15:10:42.852799278Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/*.log +ts=2024-04-16T15:10:42.852780068Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log +ts=2024-04-16T15:10:42.852742378Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log +ts=2024-04-16T15:10:42.852725798Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log +ts=2024-04-16T15:10:42.852704968Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.852619797Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}" +ts=2024-04-16T15:10:42.852585297Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hgrun/*.log +ts=2024-04-16T15:10:42.852563836Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log +ts=2024-04-16T15:10:42.852542956Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log +ts=2024-04-16T15:10:42.852526406Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log +ts=2024-04-16T15:10:42.852503976Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.852335605Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}" +ts=2024-04-16T15:10:42.852191634Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/grafana/*.log +ts=2024-04-16T15:10:42.852171694Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log +ts=2024-04-16T15:10:42.852154564Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log +ts=2024-04-16T15:10:42.852139114Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log +ts=2024-04-16T15:10:42.852119214Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/grafana/0.log reason=null +ts=2024-04-16T15:10:42.852054393Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}" +ts=2024-04-16T15:10:42.851876072Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.851851762Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.851827431Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.851787641Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.851700581Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-67b899f4c-6jkpm_97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.85160466Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*97381017-8002-430b-84fc-ed98c0becff1/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i9999\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org10\", plan=\"free\", pod=\"org10-grafana-67b899f4c-6jkpm\", pod_template_hash=\"67b899f4c\", resource_version=\"167379037\", slug=\"org10\", stackId=\"101010\"}" +ts=2024-04-16T15:10:42.844520154Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log to appear..." +ts=2024-04-16T15:10:42.84425242Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hgrun/0.log ..." +ts=2024-04-16T15:10:42.843430857Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log op=CREATE +ts=2024-04-16T15:10:42.843006913Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.839982384Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-150912.gz op=CREATE +ts=2024-04-16T15:10:42.838989799Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.8353956Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.834326829Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_distributor-59f8b8d6f-cf5s5_e3faafed-b644-460f-aa64-0c5195eb2fac/distributor/0.log.20240416-150319.tmp op=CREATE +ts=2024-04-16T15:10:42.834328514Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.831554341Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.816059502Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.805204253Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/grafana/0.log ..." +ts=2024-04-16T15:10:42.82002393Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.813772951Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log" +ts=2024-04-16T15:10:42.813717925Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log ..." +ts=2024-04-16T15:10:42.81324777Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.812324173Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-04_graphite-write-proxy-5c4768b4d-694rv_5159bfec-768a-4597-b065-9f39abfbc85d/graphite-write-proxy/0.log.20240416-150912.tmp op=CREATE +ts=2024-04-16T15:10:42.801709333Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/*.log +ts=2024-04-16T15:10:42.801692403Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log +ts=2024-04-16T15:10:42.801677883Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log +ts=2024-04-16T15:10:42.801666533Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log +ts=2024-04-16T15:10:42.801649933Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.801570113Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}" +ts=2024-04-16T15:10:42.801410661Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/*.log +ts=2024-04-16T15:10:42.801392301Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log +ts=2024-04-16T15:10:42.80136025Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log +ts=2024-04-16T15:10:42.80134193Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log +ts=2024-04-16T15:10:42.80130589Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.80124087Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}" +ts=2024-04-16T15:10:42.801144688Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.801130058Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.801111398Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.801092908Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.801059578Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.800965347Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}" +ts=2024-04-16T15:10:42.800900727Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/*.log +ts=2024-04-16T15:10:42.800843615Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log +ts=2024-04-16T15:10:42.800805775Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log +ts=2024-04-16T15:10:42.800785075Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log +ts=2024-04-16T15:10:42.800761935Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-55fdb5b7bc-npz4b_bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/0.log reason=null +ts=2024-04-16T15:10:42.800648554Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*bbadc1a8-4cdf-46a4-bbf4-16a0c08b10d1/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1010\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org11\", plan=\"free\", pod=\"org11-grafana-55fdb5b7bc-npz4b\", pod_template_hash=\"55fdb5b7bc\", resource_version=\"167301760\", slug=\"org11\", stackId=\"111111\"}" +ts=2024-04-16T15:10:42.794244605Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log op=CREATE +ts=2024-04-16T15:10:42.793880616Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.800115014Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.786968877Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.780224258Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.780195748Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/grafana/0.log ..." +ts=2024-04-16T15:10:42.780195348Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:42.780186827Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hgrun/0.log to appear..." +ts=2024-04-16T15:10:42.780165857Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hg-plugins/0.log ..." +ts=2024-04-16T15:10:42.780151287Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hgrun/0.log ..." +ts=2024-04-16T15:10:42.780050945Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.779989823Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-76c955c777-glpnp_8e9456f5-b97c-4b8e-b602-0354fc36abc5/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.779827758Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log op=CREATE +ts=2024-04-16T15:10:42.778158862Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151022.gz op=CREATE +ts=2024-04-16T15:10:42.777863082Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.777503455Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-144237.gz op=CREATE +ts=2024-04-16T15:10:42.768846823Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hgrun/*.log +ts=2024-04-16T15:10:42.768825903Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log +ts=2024-04-16T15:10:42.768805432Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log +ts=2024-04-16T15:10:42.768782331Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log +ts=2024-04-16T15:10:42.768724271Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.76861648Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}" +ts=2024-04-16T15:10:42.768356606Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.768267666Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.768230715Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.768212805Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.768176105Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.768095804Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}" +ts=2024-04-16T15:10:42.768020902Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/grafana/*.log +ts=2024-04-16T15:10:42.767966672Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log +ts=2024-04-16T15:10:42.767931071Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log +ts=2024-04-16T15:10:42.767895471Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log +ts=2024-04-16T15:10:42.767873841Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/grafana/0.log reason=null +ts=2024-04-16T15:10:42.76773819Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}" +ts=2024-04-16T15:10:42.767444506Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/*.log +ts=2024-04-16T15:10:42.767415776Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log +ts=2024-04-16T15:10:42.767389305Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log +ts=2024-04-16T15:10:42.767367185Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log +ts=2024-04-16T15:10:42.767323154Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.768149996Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.767208786Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:42.766909734Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-67d4884694-zttck_be16c84b-12ba-4590-8e1d-22bbf77c01a4/hg-plugins/0.log ..." +ts=2024-04-16T15:10:42.766936439Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log +ts=2024-04-16T15:10:42.766841898Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1212\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org12\", plan=\"free\", pod=\"org12-grafana-54444b9976-ttcfk\", pod_template_hash=\"54444b9976\", resource_version=\"167262602\", slug=\"org12\", stackId=\"121212\"}" +ts=2024-04-16T15:10:42.749268494Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:42.749193363Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-54444b9976-ttcfk_045e328e-c799-47f0-80ed-193ee19a7618/hg-plugins/0.log ..." +ts=2024-04-16T15:10:42.749456785Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.728605579Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.728528002Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.73359549Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log op=CREATE +ts=2024-04-16T15:10:42.733215247Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.728105968Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/gr-prod-03_graphite-querier-644cb88c5-k6mbl_bba9f1fd-60ff-432d-a788-19cd2e04af92/querier/0.log.20240416-151012.gz op=CREATE +ts=2024-04-16T15:10:42.729086881Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_query-frontend-9b95769c4-88ptz_c65aaf2a-b17d-43d7-ae47-592a90086d0e/query-frontend/2.log.20240416-150912.tmp op=CREATE +ts=2024-04-16T15:10:42.72344954Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.723192811Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65c44c9895-6zvtx_a12648d7-6689-4c79-9831-c366cce8d104/grafana/0.log ..." +ts=2024-04-16T15:10:42.723512942Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log op=CREATE +ts=2024-04-16T15:10:42.722517505Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.72053387Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log op=CREATE +ts=2024-04-16T15:10:42.720198477Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.719530886Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-150752.gz op=CREATE +ts=2024-04-16T15:10:42.718029955Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-145001.gz op=CREATE +ts=2024-04-16T15:10:42.712336239Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-d4f89f5f4-wvb7x_2c16ac56-b4fb-42db-b091-5feaafc58aae/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.712265048Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-d4f89f5f4-wvb7x_2c16ac56-b4fb-42db-b091-5feaafc58aae/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.716568736Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.716271215Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.713422105Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.713328587Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.713234075Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.713174528Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/grafana/0.log ..." +ts=2024-04-16T15:10:42.71291811Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.711689109Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.710581512Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.70678986Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.703742148Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.702399352Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.702151739Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.699127208Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.699053206Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/grafana/0.log ..." +ts=2024-04-16T15:10:42.692820048Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/grafana/*.log +ts=2024-04-16T15:10:42.692785128Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}" +ts=2024-04-16T15:10:42.692719938Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/*.log +ts=2024-04-16T15:10:42.692680177Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}" +ts=2024-04-16T15:10:42.692596936Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hgrun/*.log +ts=2024-04-16T15:10:42.692534796Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}" +ts=2024-04-16T15:10:42.692373454Z caller=filetarget.go:192 level=info component=logs logs_config=default msg="filetarget: watcher closed, tailer stopped, positions saved" path=/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/*.log +ts=2024-04-16T15:10:42.692271523Z caller=filetargetmanager.go:397 level=info component=logs logs_config=default msg="Removing target" key="/var/log/pods/*32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1313\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"org13\", plan=\"free\", pod=\"org13-grafana-5c8b54fd6c-9lp4b\", pod_template_hash=\"5c8b54fd6c\", resource_version=\"143617487\", slug=\"org13\", stackId=\"131313\"}" +ts=2024-04-16T15:10:42.691652875Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:42.691549847Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7b5df5b97b-chz7b_c41f2190-4e4f-4361-b8ff-56c9a1e1bcf9/hg-plugins/0.log ..." +ts=2024-04-16T15:10:42.689356411Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log to appear..." +ts=2024-04-16T15:10:42.68929355Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-cccfb8fd6-cktl7_bd6a9960-6f03-46ee-b646-1456cff50d71/hgrun/0.log ..." +ts=2024-04-16T15:10:42.681178579Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log to appear..." +ts=2024-04-16T15:10:42.681073898Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-7fd6786f4b-242cb_404e8595-2e9f-4fcf-9495-925f6d245e20/hg-plugins/0.log ..." +ts=2024-04-16T15:10:42.684055816Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_distributor-76c559cb69-9s6gv_e844bf19-fdc1-4b1a-8eaa-59338915688c/distributor/0.log" +ts=2024-04-16T15:10:42.683990983Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_distributor-76c559cb69-9s6gv_e844bf19-fdc1-4b1a-8eaa-59338915688c/distributor/0.log ..." +ts=2024-04-16T15:10:42.68284694Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.680210253Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log +ts=2024-04-16T15:10:42.680182593Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log +ts=2024-04-16T15:10:42.680157683Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log +ts=2024-04-16T15:10:42.680129951Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log reason=null +ts=2024-04-16T15:10:42.680039191Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log +ts=2024-04-16T15:10:42.679805009Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun +ts=2024-04-16T15:10:42.677990299Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hgrun/0.log +ts=2024-04-16T15:10:42.677695535Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log op=CREATE +ts=2024-04-16T15:10:42.67757253Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log" +ts=2024-04-16T15:10:42.677525677Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log ..." +ts=2024-04-16T15:10:42.677253034Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.673854007Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod_parallel-querier-54566cbb6c-d5tbl_b5e4705e-f54a-44c2-8e3b-9d6417be4208/parallel-querier/0.log.20240416-150950.gz op=CREATE +ts=2024-04-16T15:10:42.673361577Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.606839ms +ts=2024-04-16T15:10:42.673347703Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=12.306545ms +ts=2024-04-16T15:10:42.673335684Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.582085ms +ts=2024-04-16T15:10:42.673321039Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.570803ms +ts=2024-04-16T15:10:42.67330676Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.500669ms +ts=2024-04-16T15:10:42.673292934Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.557649ms +ts=2024-04-16T15:10:42.673273766Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.549069ms +ts=2024-04-16T15:10:42.673260645Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.416831ms +ts=2024-04-16T15:10:42.673245436Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=4.29634ms +ts=2024-04-16T15:10:42.673229231Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=18.413586ms +ts=2024-04-16T15:10:42.67319054Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=10.120498ms +ts=2024-04-16T15:10:42.67317492Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=8.245874ms +ts=2024-04-16T15:10:42.673158204Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=10.098523ms +ts=2024-04-16T15:10:42.673140834Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=10.06335ms +ts=2024-04-16T15:10:42.673108764Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=10.051817ms +ts=2024-04-16T15:10:42.673076835Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.877726ms +ts=2024-04-16T15:10:42.673058533Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.882173ms +ts=2024-04-16T15:10:42.67304026Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.263441ms +ts=2024-04-16T15:10:42.673013718Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.986533ms +ts=2024-04-16T15:10:42.672994786Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.232997ms +ts=2024-04-16T15:10:42.672984226Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.130626ms +ts=2024-04-16T15:10:42.672971725Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.171497ms +ts=2024-04-16T15:10:42.672959904Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.176985ms +ts=2024-04-16T15:10:42.672942987Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.084013ms +ts=2024-04-16T15:10:42.672926073Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=11.896444ms +ts=2024-04-16T15:10:42.672910999Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=3.998797ms +ts=2024-04-16T15:10:42.672894776Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=11.850326ms +ts=2024-04-16T15:10:42.672894249Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=14.171954ms +ts=2024-04-16T15:10:42.672866028Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.086994ms +ts=2024-04-16T15:10:42.67283067Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.096322ms +ts=2024-04-16T15:10:42.672814991Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.058304ms +ts=2024-04-16T15:10:42.670517273Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.032523ms +ts=2024-04-16T15:10:42.668933743Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=36.208371ms +ts=2024-04-16T15:10:42.66886626Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=36.05808ms +ts=2024-04-16T15:10:42.667280183Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.667255583Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.667241643Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.667227123Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log reason=null +ts=2024-04-16T15:10:42.667185182Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.667135012Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security +ts=2024-04-16T15:10:42.665722097Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.664904534Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.781937ms +ts=2024-04-16T15:10:42.663064079Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log +ts=2024-04-16T15:10:42.663029038Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log +ts=2024-04-16T15:10:42.662977628Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log +ts=2024-04-16T15:10:42.662936316Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log reason=null +ts=2024-04-16T15:10:42.662857596Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log +ts=2024-04-16T15:10:42.662782495Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins +ts=2024-04-16T15:10:42.663057068Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.044531ms +ts=2024-04-16T15:10:42.663048272Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=32.234907ms +ts=2024-04-16T15:10:42.66304184Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=13.994973ms +ts=2024-04-16T15:10:42.663029813Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=50.032359ms +ts=2024-04-16T15:10:42.661585692Z caller=tailer.go:245 level=info component=logs logs_config=default component=tailer msg="stopped tailing file" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log +ts=2024-04-16T15:10:42.661559292Z caller=tailer.go:118 level=info component=logs logs_config=default component=tailer msg="position timer: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log +ts=2024-04-16T15:10:42.661516881Z caller=tailer.go:155 level=info component=logs logs_config=default component=tailer msg="tail routine: exited" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log +ts=2024-04-16T15:10:42.661496161Z caller=tailer.go:164 level=info component=logs logs_config=default component=tailer msg="tail routine: tail channel closed, stopping tailer" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log reason=null +ts=2024-04-16T15:10:42.661448091Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log +ts=2024-04-16T15:10:42.66137395Z caller=filetarget.go:326 level=info component=logs logs_config=default msg="removing directory from watcher" directory=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana +ts=2024-04-16T15:10:42.661240579Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/hg-plugins/0.log +ts=2024-04-16T15:10:42.661012758Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=40.387272ms +ts=2024-04-16T15:10:42.661004266Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.237323ms +ts=2024-04-16T15:10:42.660997233Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=32.218555ms +ts=2024-04-16T15:10:42.659763033Z caller=tailer.go:207 level=info component=logs logs_config=default component=tailer msg="skipping update of position for a file which does not currently exist" path=/var/log/pods/hosted-grafana_.something-grafana-5c8b54fd6c-9lp4b_32bb6165-f27e-4160-8068-54f665f5b240/grafana/0.log +ts=2024-04-16T15:10:42.658807921Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=39.820002ms +ts=2024-04-16T15:10:42.658702944Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.742622ms +ts=2024-04-16T15:10:42.658726236Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=31.767787ms +ts=2024-04-16T15:10:42.658714221Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=35.841792ms +ts=2024-04-16T15:10:42.658714776Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.927979ms +ts=2024-04-16T15:10:42.658695601Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.81259ms +ts=2024-04-16T15:10:42.65866213Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.858297ms +ts=2024-04-16T15:10:42.654781676Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=41.792949ms +ts=2024-04-16T15:10:42.653169395Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.091185ms +ts=2024-04-16T15:10:42.651144688Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.288699ms +ts=2024-04-16T15:10:42.651057845Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.125234ms +ts=2024-04-16T15:10:42.649021734Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.269305ms +ts=2024-04-16T15:10:42.648955137Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.689603ms +ts=2024-04-16T15:10:42.644901985Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.188226ms +ts=2024-04-16T15:10:42.642995943Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.294935ms +ts=2024-04-16T15:10:42.640846468Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=37.01759ms +ts=2024-04-16T15:10:42.640837803Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.015617ms +ts=2024-04-16T15:10:42.640745996Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.018571ms +ts=2024-04-16T15:10:42.640760648Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.027827ms +ts=2024-04-16T15:10:42.640752425Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.427749ms +ts=2024-04-16T15:10:42.640753435Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=55.809823ms +ts=2024-04-16T15:10:42.640738991Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=42.238747ms +ts=2024-04-16T15:10:42.640721969Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=44.181487ms +ts=2024-04-16T15:10:42.640415221Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-65649b6fd6-4fxxr_02a8ddd6-b563-4890-a8d5-8b53ba00111b/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.639945597Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-65649b6fd6-4fxxr_02a8ddd6-b563-4890-a8d5-8b53ba00111b/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.636702915Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=42.405442ms +ts=2024-04-16T15:10:42.634457562Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=37.933939ms +ts=2024-04-16T15:10:42.632745858Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.592749ms +ts=2024-04-16T15:10:42.632704401Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=36.257142ms +ts=2024-04-16T15:10:42.632702581Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=41.462693ms +ts=2024-04-16T15:10:42.630777393Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=52.719537ms +ts=2024-04-16T15:10:42.630640151Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/otlp-gateway_otlp-gateway-7869b8fc46-ggpjr_d51b35fe-fa02-42cc-9587-ba9e6cf2a372/otlp-gateway/0.log" +ts=2024-04-16T15:10:42.630556426Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/otlp-gateway_otlp-gateway-7869b8fc46-ggpjr_d51b35fe-fa02-42cc-9587-ba9e6cf2a372/otlp-gateway/0.log ..." +ts=2024-04-16T15:10:42.629476701Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.628748536Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.898809ms +ts=2024-04-16T15:10:42.627663582Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod-011_querier-burst-67f76b84bf-jp5r2_0fb2bd0f-7fd3-42a1-a2bc-f4b1173dca16/querier/0.log.20240416-150752.tmp op=CREATE +ts=2024-04-16T15:10:42.613959129Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-10_ingester-zone-c-116_4655218c-833b-460a-879b-c5ccc57b70c4/ingester/0.log.20240416-144237.tmp op=CREATE +ts=2024-04-16T15:10:42.626926123Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=41.961295ms +ts=2024-04-16T15:10:42.624776836Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.828771ms +ts=2024-04-16T15:10:42.624736788Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=39.814769ms +ts=2024-04-16T15:10:42.622840931Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.933798ms +ts=2024-04-16T15:10:42.620593182Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=42.477123ms +ts=2024-04-16T15:10:42.620515824Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.618872355Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.789061ms +ts=2024-04-16T15:10:42.618873556Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.76289ms +ts=2024-04-16T15:10:42.618845099Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=35.695961ms +ts=2024-04-16T15:10:42.61677274Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=38.670924ms +ts=2024-04-16T15:10:42.613055386Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.959848ms +ts=2024-04-16T15:10:42.61296789Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.845608ms +ts=2024-04-16T15:10:42.612956175Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=34.849789ms +ts=2024-04-16T15:10:42.612326411Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/grafana/0.log to appear..." +ts=2024-04-16T15:10:42.611233748Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.140862ms +ts=2024-04-16T15:10:42.61085982Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/grafana/0.log ..." +ts=2024-04-16T15:10:42.609822629Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.748642ms +ts=2024-04-16T15:10:42.608846393Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-6vkzk_69ffa46d-5698-4d3a-aa3e-a218bddab1e7/cortex-gw/0.log" +ts=2024-04-16T15:10:42.608739764Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/gr-prod-04_cortex-gw-6579fd68cf-6vkzk_69ffa46d-5698-4d3a-aa3e-a218bddab1e7/cortex-gw/0.log ..." +ts=2024-04-16T15:10:42.60772791Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.664548ms +ts=2024-04-16T15:10:42.607682174Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.589004ms +ts=2024-04-16T15:10:42.607073897Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/cortex-prod-06_ruler-586cbc8869-n7g68_7127c2b9-28a3-4111-b151-4480358a1638/ruler/0.log" +ts=2024-04-16T15:10:42.606988671Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/cortex-prod-06_ruler-586cbc8869-n7g68_7127c2b9-28a3-4111-b151-4480358a1638/ruler/0.log ..." +ts=2024-04-16T15:10:42.603802514Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=18.738931ms +ts=2024-04-16T15:10:42.603794604Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.717986ms +ts=2024-04-16T15:10:42.603705733Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=25.680015ms +ts=2024-04-16T15:10:42.603664662Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=18.695855ms +ts=2024-04-16T15:10:42.603661227Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=25.564163ms +ts=2024-04-16T15:10:42.601446876Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/loki-prod3_parallel-querier-burst-68d978b6bd-tqtt7_53706fa4-7ae2-4000-9905-5c2289ffcb80/parallel-querier/0.log.20240416-145001.tmp op=CREATE +ts=2024-04-16T15:10:42.600671271Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.600294563Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=22.235405ms +ts=2024-04-16T15:10:42.598467331Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.178172ms +ts=2024-04-16T15:10:42.596495024Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.632352ms +ts=2024-04-16T15:10:42.596487701Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=19.632253ms +ts=2024-04-16T15:10:42.596411984Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=28.429057ms +ts=2024-04-16T15:10:42.59426318Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_92276_cloudwatch_tags duration=16.22324ms +ts=2024-04-16T15:10:42.591096003Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=2.040881ms +ts=2024-04-16T15:10:42.591104598Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.034134ms +ts=2024-04-16T15:10:42.589029314Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.10715ms +ts=2024-04-16T15:10:42.586876248Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.586817954Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=16.887584ms +ts=2024-04-16T15:10:42.586376081Z caller=log.go:168 component=logs logs_config=default level=info msg="Successfully reopened /var/log/pods/loki-prod3_querier-688cf9ff66-ckh64_88063351-e149-431f-806d-52987df865ca/querier/0.log" +ts=2024-04-16T15:10:42.586278511Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/loki-prod3_querier-688cf9ff66-ckh64_88063351-e149-431f-806d-52987df865ca/querier/0.log ..." +ts=2024-04-16T15:10:42.58487305Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=16.89909ms +ts=2024-04-16T15:10:42.584928018Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.98178ms +ts=2024-04-16T15:10:42.584921953Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.178899ms +ts=2024-04-16T15:10:42.584925445Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=16.858537ms +ts=2024-04-16T15:10:42.584913112Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.920848ms +ts=2024-04-16T15:10:42.584894716Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.098321ms +ts=2024-04-16T15:10:42.584867384Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=16.892285ms +ts=2024-04-16T15:10:42.584760426Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:42.584727732Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node/0.log +ts=2024-04-16T15:10:42.584643207Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/kube-system_calico-node-nbpjs_9c0b6492-afdc-4657-b99d-a7503d2fcb00/calico-node +ts=2024-04-16T15:10:42.583982669Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 93 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.583107672Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.504842ms +ts=2024-04-16T15:10:42.581700829Z caller=log.go:168 component=logs logs_config=default level=info msg="Waiting for /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/hosted-grafana-security/0.log to appear..." +ts=2024-04-16T15:10:42.581418535Z caller=log.go:168 component=logs logs_config=default level=info msg="Re-opening moved/deleted file /var/log/pods/hosted-grafana_.something-grafana-5796686bc-lwgkl_6f827ff8-26e9-43ee-b131-31037198f332/hosted-grafana-security/0.log ..." +ts=2024-04-16T15:10:42.57663139Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 94 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.576603054Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.196731ms +ts=2024-04-16T15:10:42.576588908Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=13.068655ms +ts=2024-04-16T15:10:42.576571188Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=2.277991ms +ts=2024-04-16T15:10:42.576553041Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.109411ms +ts=2024-04-16T15:10:42.576537264Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=4.424369ms +ts=2024-04-16T15:10:42.576491264Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=8.515501ms +ts=2024-04-16T15:10:42.576522639Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=15.178204ms +ts=2024-04-16T15:10:42.576504949Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.470666ms +ts=2024-04-16T15:10:42.576458896Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.195099ms +ts=2024-04-16T15:10:42.576470384Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=6.413532ms +ts=2024-04-16T15:10:42.576461807Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=15.098093ms +ts=2024-04-16T15:10:42.576436074Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.129507ms +ts=2024-04-16T15:10:42.576432522Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=35.78072ms +ts=2024-04-16T15:10:42.576423296Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=19.320399ms +ts=2024-04-16T15:10:42.576433546Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=21.127352ms +ts=2024-04-16T15:10:42.57642251Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=17.134007ms +ts=2024-04-16T15:10:42.576403643Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=24.447434ms +ts=2024-04-16T15:10:42.576332073Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=26.680906ms +ts=2024-04-16T15:10:42.576321491Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.457388ms +ts=2024-04-16T15:10:42.576310653Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=17.008867ms +ts=2024-04-16T15:10:42.57627133Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=26.554435ms +ts=2024-04-16T15:10:42.576029821Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/grafana-ruler_grafana-ruler-78bc75f5d6-p98cc_e082d5d3-bb47-435b-9c42-9304ea43d9a5/grafana-ruler/0.log.20240416-151022.tmp op=CREATE +ts=2024-04-16T15:10:42.574265315Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.719897ms +ts=2024-04-16T15:10:42.573530517Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log op=CREATE +ts=2024-04-16T15:10:42.573182782Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log.20240416-151042 op=CREATE +ts=2024-04-16T15:10:42.572085566Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=22.417183ms +ts=2024-04-16T15:10:42.565828672Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/0.log +ts=2024-04-16T15:10:42.565798661Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:42.565656449Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins +ts=2024-04-16T15:10:42.564661119Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/0.log +ts=2024-04-16T15:10:42.564618808Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:42.564392566Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/0.log +ts=2024-04-16T15:10:42.564308205Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:42.564046973Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security +ts=2024-04-16T15:10:42.571355512Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="bufio.Scanner: token too long" +ts=2024-04-16T15:10:42.569999783Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.446773ms +ts=2024-04-16T15:10:42.569966615Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=22.473028ms +ts=2024-04-16T15:10:42.569895328Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.619424ms +ts=2024-04-16T15:10:42.569919882Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.276926ms +ts=2024-04-16T15:10:42.569907592Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.29022ms +ts=2024-04-16T15:10:42.569892451Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.350764ms +ts=2024-04-16T15:10:42.569343595Z caller=filetargetmanager.go:181 level=info component=logs logs_config=default msg="received file watcher event" name=/var/log/pods/cortex-prod-06_ruler-586cbc8869-f8n6l_edecd605-8103-49b9-8c74-ea64d43fff32/ruler/0.log.20240416-151002.gz op=CREATE +ts=2024-04-16T15:10:42.56856052Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.567938235Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.39652ms +ts=2024-04-16T15:10:42.568026014Z caller=logfmt.go:139 level=error component=logs logs_config=default component=file_pipeline component=stage type=logfmt msg="failed to decode logfmt" err="logfmt syntax error at pos 95 on line 1: unexpected '\"'" +ts=2024-04-16T15:10:42.567957297Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.431728ms +ts=2024-04-16T15:10:42.567953817Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.298132ms +ts=2024-04-16T15:10:42.567957895Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=27.448889ms +ts=2024-04-16T15:10:42.56793769Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=29.70719ms +ts=2024-04-16T15:10:42.563669017Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=51.443269ms +ts=2024-04-16T15:10:42.563592311Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.019732ms +ts=2024-04-16T15:10:42.563619118Z caller=tailer.go:147 level=info component=logs logs_config=default component=tailer msg="tail routine: started" path=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/0.log +ts=2024-04-16T15:10:42.563576238Z caller=log.go:168 component=logs logs_config=default level=info msg="Seeked /var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/0.log - &{Offset:0 Whence:0}" +ts=2024-04-16T15:10:42.563404315Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/grafana +ts=2024-04-16T15:10:42.56347333Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=35.794868ms +ts=2024-04-16T15:10:42.56199166Z caller=filetarget.go:313 level=info component=logs logs_config=default msg="watching new directory" directory=/var/log/pods/hosted-grafana_.something-grafana-7c65678f86-9zhlb_b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun +ts=2024-04-16T15:10:42.561385592Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.672584ms +ts=2024-04-16T15:10:42.561572233Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=37.96656ms +ts=2024-04-16T15:10:42.561408375Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=44.74634ms +ts=2024-04-16T15:10:42.561376386Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=25.453761ms +ts=2024-04-16T15:10:42.561342238Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=29.564864ms +ts=2024-04-16T15:10:42.561317318Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=31.684575ms +ts=2024-04-16T15:10:42.557102408Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hosted-grafana-security/*.log:{app=\"grafana\", conprof=\"true\", container=\"hosted-grafana-security\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}" +ts=2024-04-16T15:10:42.556930066Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hg-plugins/*.log:{app=\"grafana\", conprof=\"true\", container=\"hg-plugins\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}" +ts=2024-04-16T15:10:42.556706613Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/hgrun/*.log:{app=\"grafana\", conprof=\"true\", container=\"hgrun\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}" +ts=2024-04-16T15:10:42.556278698Z caller=filetargetmanager.go:361 level=info component=logs logs_config=default msg="Adding target" key="/var/log/pods/*b92ee988-5c26-4c64-bba3-ff6a01723759/grafana/*.log:{app=\"grafana\", conprof=\"true\", container=\"grafana\", instanceId=\"i1111\", job=\"hosted-grafana/grafana\", name=\"grafana\", namespace=\"hosted-grafana\", org=\"orgnamehere\", plan=\"free\", pod=\"orgnamehere-grafana-7c65678f86-9zhlb\", pod_template_hash=\"7c65678f86\", resource_version=\"143638246\", slug=\"orgnamehere\", stackId=\"866772\"}" +ts=2024-04-16T15:10:42.559283249Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=33.523804ms +ts=2024-04-16T15:10:42.559263621Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_tags duration=40.654994ms +ts=2024-04-16T15:10:42.557068004Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=40.414993ms +ts=2024-04-16T15:10:42.55528547Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.748545ms +ts=2024-04-16T15:10:42.555273483Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=14.735015ms +ts=2024-04-16T15:10:42.555218448Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.592855ms +2024-04-16 15:10:42.555 ts=2024-04-16T15:10:42.555230437Z level=info msg="finished node evaluation" controller_id=module.http.cloudwatch_pipelines node_id=prometheus.scrape.stack_378175_cloudwatch_notags duration=38.545339ms \ No newline at end of file diff --git a/pkg/pattern/drain/testdata/custom.txt b/pkg/pattern/drain/testdata/custom.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pkg/pattern/drain/testdata/drone-json.txt b/pkg/pattern/drain/testdata/drone-json.txt new file mode 100644 index 000000000000..2f6862236cf1 --- /dev/null +++ b/pkg/pattern/drain/testdata/drone-json.txt @@ -0,0 +1,210 @@ +{"duration":20950259,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:03:28Z"} +{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:03:28Z"} +{"id":"q62wCcIkEOueqFKF","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:03:28Z"} +{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:03:28Z"} +{"id":"q62wCcIkEOueqFKF","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:03:28Z"} +{"duration":24479472,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":56560748,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:03:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:02:58Z"} +{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:02:58Z"} +{"id":"m6SpYHzdXrDAFqDR","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:58Z"} +{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:02:58Z"} +{"id":"m6SpYHzdXrDAFqDR","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:02:58Z"} +{"duration":36437134,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":36704414,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:02:28Z"} +{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:02:28Z"} +{"id":"T0I8Dsnw3uSi3Gal","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:02:28Z"} +{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:02:28Z"} +{"id":"T0I8Dsnw3uSi3Gal","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:02:28Z"} +{"duration":49120197,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":69764560,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:02:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:01:58Z"} +{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:01:58Z"} +{"id":"9eA72xOtx8kzMhXn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:58Z"} +{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:01:58Z"} +{"id":"9eA72xOtx8kzMhXn","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:01:58Z"} +{"duration":93395725,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":27353687,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:01:28Z"} +{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:01:28Z"} +{"id":"pet7QVfO1yE8fk56","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:01:28Z"} +{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:01:27Z"} +{"id":"pet7QVfO1yE8fk56","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:01:27Z"} +{"duration":21783864,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":27700228,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:01:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:00:57Z"} +{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:00:57Z"} +{"id":"15eSzaEG0enf86Kl","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:57Z"} +{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:00:57Z"} +{"id":"15eSzaEG0enf86Kl","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:00:57Z"} +{"duration":20800183,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":114305939,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"check capacity complete","time":"2024-04-16T15:00:27Z"} +{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"no capacity changes required","time":"2024-04-16T15:00:27Z"} +{"id":"JO1OT5ADoNA8NYqr","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T15:00:27Z"} +{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"calculate server capacity","time":"2024-04-16T15:00:27Z"} +{"id":"JO1OT5ADoNA8NYqr","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T15:00:27Z"} +{"duration":118009488,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":33663734,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T15:00:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:59:57Z"} +{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:59:57Z"} +{"id":"Xz2OCJhgeBSRFyoN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:57Z"} +{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:59:57Z"} +{"id":"Xz2OCJhgeBSRFyoN","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:59:57Z"} +{"duration":48520224,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":25172573,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:59:27Z"} +{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:59:27Z"} +{"id":"pPc2ORUhHAhFgBg3","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:59:27Z"} +{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:59:27Z"} +{"id":"pPc2ORUhHAhFgBg3","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:59:27Z"} +{"duration":215980677,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":58239998,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:59:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:58:57Z"} +{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:58:57Z"} +{"id":"4G6Srn6lSwzYrx19","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:57Z"} +{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:58:57Z"} +{"id":"4G6Srn6lSwzYrx19","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:58:57Z"} +{"duration":33198841,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":38097220,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:58:27Z"} +{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:58:27Z"} +{"id":"1Lu90T1fWzsWOKlc","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:58:27Z"} +{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:58:27Z"} +{"id":"1Lu90T1fWzsWOKlc","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:58:27Z"} +{"duration":63956469,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":125762940,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:58:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:57:57Z"} +{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:57:57Z"} +{"id":"4XjwwNoOwZFaWePQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:57Z"} +{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:57:57Z"} +{"id":"4XjwwNoOwZFaWePQ","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:57:56Z"} +{"duration":33644691,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":24328424,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"IQy23J3NON0BV10V","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:57:26Z"} +{"id":"IQy23J3NON0BV10V","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:57:26Z"} +{"id":"IQy23J3NON0BV10V","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:57:26Z"} +{"id":"IQy23J3NON0BV10V","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:57:26Z"} +{"id":"IQy23J3NON0BV10V","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:57:26Z"} +{"duration":20257078,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":28190860,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:57:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:56:56Z"} +{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:56:56Z"} +{"id":"FQ8wCQfaR9W387cH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:56Z"} +{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:56:56Z"} +{"id":"FQ8wCQfaR9W387cH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:56:56Z"} +{"duration":124540195,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":33392901,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:56:26Z"} +{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:56:26Z"} +{"id":"Hhwn7ecXjxF67DG6","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:56:26Z"} +{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:56:26Z"} +{"id":"Hhwn7ecXjxF67DG6","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:56:26Z"} +{"duration":38704940,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":41929762,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:56:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:55:56Z"} +{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:55:56Z"} +{"id":"luflyGZvZnLzhQEH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:56Z"} +{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:55:56Z"} +{"id":"luflyGZvZnLzhQEH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:55:56Z"} +{"duration":20307958,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":20185884,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:55:26Z"} +{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:55:26Z"} +{"id":"q20GZcvyzMwrTGx5","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:55:26Z"} +{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:55:26Z"} +{"id":"q20GZcvyzMwrTGx5","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:55:26Z"} +{"duration":62732272,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":233876906,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:55:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:54:56Z"} +{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:54:56Z"} +{"id":"3K61Yf6ImKYexoFx","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:56Z"} +{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:54:56Z"} +{"id":"3K61Yf6ImKYexoFx","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:54:53Z"} +{"duration":91919807,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":123141860,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:54:23Z"} +{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:54:23Z"} +{"id":"SmbOO0l5aADX9BaQ","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:54:23Z"} +{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:54:23Z"} +{"id":"SmbOO0l5aADX9BaQ","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:54:23Z"} +{"duration":127118328,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":154732993,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:54:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:53:53Z"} +{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:53:53Z"} +{"id":"96TvvsMzSkkaW8oW","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:53Z"} +{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:53:53Z"} +{"id":"96TvvsMzSkkaW8oW","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:53:53Z"} +{"duration":118361504,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":137758190,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:53:23Z"} +{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:53:23Z"} +{"id":"C7aYn8cb4NCrkkYI","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:53:23Z"} +{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:53:23Z"} +{"id":"C7aYn8cb4NCrkkYI","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:53:23Z"} +{"duration":68794582,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":74435188,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:53:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:52:53Z"} +{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:52:53Z"} +{"id":"CMG7ZwwYqNPBonAn","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:53Z"} +{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:52:53Z"} +{"id":"CMG7ZwwYqNPBonAn","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:52:53Z"} +{"duration":50623103,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":55970830,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:33Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"focV9BzODwRbWwKE","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:52:23Z"} +{"id":"focV9BzODwRbWwKE","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:52:23Z"} +{"id":"focV9BzODwRbWwKE","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:52:23Z"} +{"id":"focV9BzODwRbWwKE","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:52:23Z"} +{"id":"focV9BzODwRbWwKE","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:52:23Z"} +{"duration":33285760,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":18306142,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:52:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:51:53Z"} +{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:51:53Z"} +{"id":"HphRnJOM8uYohf1p","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:53Z"} +{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:51:53Z"} +{"id":"HphRnJOM8uYohf1p","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:51:53Z"} +{"duration":174783742,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":31536414,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:51:23Z"} +{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:51:23Z"} +{"id":"m3n8GndhG45uGIQA","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:51:23Z"} +{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:51:22Z"} +{"id":"m3n8GndhG45uGIQA","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:51:22Z"} +{"duration":119507962,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:18Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":430410510,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:51:03Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:50:52Z"} +{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:50:52Z"} +{"id":"nTO38tWtnvRWRl1G","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:52Z"} +{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:50:52Z"} +{"id":"nTO38tWtnvRWRl1G","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:50:52Z"} +{"duration":23638914,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":20300837,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:50:22Z"} +{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:50:22Z"} +{"id":"5qEIzErDfiALVPAN","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:50:22Z"} +{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:50:22Z"} +{"id":"5qEIzErDfiALVPAN","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:50:22Z"} +{"duration":33318175,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":19013978,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:50:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:49:52Z"} +{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:49:52Z"} +{"id":"q61oHTtF4MMiQVGH","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:52Z"} +{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:49:52Z"} +{"id":"q61oHTtF4MMiQVGH","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:49:52Z"} +{"duration":19815267,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:47Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":21037324,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:32Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:49:22Z"} +{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:49:22Z"} +{"id":"4rNxIlhDKxGgzBHe","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:49:22Z"} +{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:49:22Z"} +{"id":"4rNxIlhDKxGgzBHe","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:49:22Z"} +{"duration":20219085,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:17Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"duration":24085452,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:49:02Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} +{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"check capacity complete","time":"2024-04-16T14:48:52Z"} +{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"no capacity changes required","time":"2024-04-16T14:48:52Z"} +{"id":"D4Oh1ivB6cdLWa08","level":"debug","max-pool":4,"min-pool":0,"msg":"check capacity","pending-builds":0,"running-builds":0,"server-buffer":0,"server-capacity":0,"server-count":0,"time":"2024-04-16T14:48:52Z"} +{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"calculate server capacity","time":"2024-04-16T14:48:52Z"} +{"id":"D4Oh1ivB6cdLWa08","level":"debug","msg":"calculate unfinished jobs","time":"2024-04-16T14:48:52Z"} +{"duration":51986975,"level":"debug","method":"GET","msg":"request completed","referer":"","remote":"10.136.105.40:52702","request":"/metrics","status":200,"time":"2024-04-16T14:48:48Z","user-agent":"GrafanaAgent/v0.40.3 (flow; linux; helm)"} \ No newline at end of file diff --git a/pkg/pattern/drain/testdata/ingester-logfmt.txt b/pkg/pattern/drain/testdata/ingester-logfmt.txt new file mode 100644 index 000000000000..3b5f18e6470b --- /dev/null +++ b/pkg/pattern/drain/testdata/ingester-logfmt.txt @@ -0,0 +1,100 @@ +ts=2024-04-17T09:52:46.364803306Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368313ms" +ts=2024-04-17T09:52:46.364804708Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.619763ms" +ts=2024-04-17T09:52:46.364730588Z caller=http.go:194 level=debug traceID=116a1a427c28ef7c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 944.101µs" +ts=2024-04-17T09:52:46.364808055Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 595.34µs" +ts=2024-04-17T09:52:46.364790582Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 845.635µs" +ts=2024-04-17T09:52:46.36478937Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 677.201µs" +ts=2024-04-17T09:52:46.36470023Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.702772ms" +ts=2024-04-17T09:52:46.364312973Z caller=http.go:194 level=debug traceID=241f9222b132d55f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.305663ms" +ts=2024-04-17T09:52:46.364492298Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 672.113µs" +ts=2024-04-17T09:52:46.36471937Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.822532ms" +ts=2024-04-17T09:52:46.364561196Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.468961ms" +ts=2024-04-17T09:52:46.36477128Z caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory +ts=2024-04-17T09:52:46.364749251Z caller=http.go:194 level=debug traceID=6b72cb766bf5191f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 8.508724ms" +ts=2024-04-17T09:52:46.364726948Z caller=head.go:216 level=debug tenant=987678 msg="profile is empty after delta computation" metricName=memory +ts=2024-04-17T09:52:46.364753592Z caller=http.go:194 level=debug traceID=4d4de2711f38c05c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.564507ms" +ts=2024-04-17T09:52:46.364737024Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 967.106µs" +ts=2024-04-17T09:52:46.364734092Z caller=http.go:194 level=debug traceID=345399fe6745a50c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 10.63998ms" +ts=2024-04-17T09:52:46.364684284Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 839.55µs" +ts=2024-04-17T09:52:46.364649689Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 783.914µs" +ts=2024-04-17T09:52:46.364650688Z caller=http.go:194 level=debug traceID=568f48b07fd4c3ef orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 874.107µs" +ts=2024-04-17T09:52:46.364616184Z caller=http.go:194 level=debug traceID=2b1bf144cf74bf28 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 17.261341ms" +ts=2024-04-17T09:52:46.364595982Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 914.739µs" +ts=2024-04-17T09:52:46.364584929Z caller=http.go:194 level=debug traceID=5b8bca4cdba21f43 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.142724ms" +ts=2024-04-17T09:52:46.364590656Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 895.782µs" +ts=2024-04-17T09:52:46.364657596Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368331ms" +ts=2024-04-17T09:52:46.364622325Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 660.247µs" +ts=2024-04-17T09:52:46.364594116Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 467.285µs" +ts=2024-04-17T09:52:46.364545621Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 530.452µs" +ts=2024-04-17T09:52:46.36450376Z caller=http.go:194 level=debug traceID=74f2e8cf7e0c7fc7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.135674ms" +ts=2024-04-17T09:52:46.36443397Z caller=http.go:194 level=debug traceID=6fb0c4a951fa6f0f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.398122ms" +ts=2024-04-17T09:52:46.364464147Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.131431ms" +ts=2024-04-17T09:52:46.364440826Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 1.023612ms" +ts=2024-04-17T09:52:46.36432296Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.408666ms" +ts=2024-04-17T09:52:46.364405022Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.341907ms" +ts=2024-04-17T09:52:46.36444769Z caller=http.go:194 level=debug traceID=568f48b07fd4c3ef orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 632.404µs" +ts=2024-04-17T09:52:46.364429591Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 478.714µs" +ts=2024-04-17T09:52:46.364341351Z caller=http.go:194 level=debug traceID=4d4de2711f38c05c orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.09688ms" +ts=2024-04-17T09:52:46.364357027Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 763.325µs" +ts=2024-04-17T09:52:46.364363024Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.501374ms" +ts=2024-04-17T09:52:46.364308133Z caller=http.go:194 level=debug traceID=6fb0c4a951fa6f0f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.102861ms" +ts=2024-04-17T09:52:46.36423841Z caller=http.go:194 level=debug traceID=071fd5ad1d0d0515 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.527951ms" +ts=2024-04-17T09:52:46.364312232Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 858.218µs" +ts=2024-04-17T09:52:46.36428579Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 492.16µs" +ts=2024-04-17T09:52:46.361802864Z caller=http.go:194 level=debug traceID=744100a66cc00de4 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.011408ms" +ts=2024-04-17T09:52:46.364289781Z caller=http.go:194 level=debug traceID=285eaac96a357b61 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 502.847µs" +ts=2024-04-17T09:52:46.364200562Z caller=http.go:194 level=debug traceID=62968dddfc09bbbe orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.494135ms" +ts=2024-04-17T09:52:46.364282918Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 704.283µs" +ts=2024-04-17T09:52:46.364279378Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 764.795µs" +ts=2024-04-17T09:52:46.364217907Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 485.357µs" +ts=2024-04-17T09:52:46.364219229Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 837.055µs" +ts=2024-04-17T09:52:46.364112209Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 679.644µs" +ts=2024-04-17T09:52:46.364155917Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.279766ms" +ts=2024-04-17T09:52:46.364081008Z caller=http.go:194 level=debug traceID=64b8a1a256934690 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.267758ms" +ts=2024-04-17T09:52:46.364189774Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 880.75µs" +ts=2024-04-17T09:52:46.364145376Z caller=http.go:194 level=debug traceID=70121da20778353f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.053066ms" +ts=2024-04-17T09:52:46.364093541Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 660.622µs" +ts=2024-04-17T09:52:46.364068342Z caller=http.go:194 level=debug traceID=3dc61701564a23f7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 5.097204ms" +ts=2024-04-17T09:52:46.36353869Z caller=http.go:194 level=debug traceID=4a7fddbe653f5299 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.804943ms" +ts=2024-04-17T09:52:46.362372357Z caller=http.go:194 level=debug traceID=0f348e7d30d12cff orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 2.009358ms" +ts=2024-04-17T09:52:46.364063967Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.368053ms" +ts=2024-04-17T09:52:46.364085113Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 918.778µs" +ts=2024-04-17T09:52:46.364063374Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 370.137µs" +ts=2024-04-17T09:52:46.358938595Z caller=http.go:194 level=debug traceID=694c1939d3e1fbc7 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 926.871µs" +ts=2024-04-17T09:52:46.364006323Z caller=http.go:194 level=debug traceID=241f9222b132d55f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.061869ms" +ts=2024-04-17T09:52:46.363969935Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 773.863µs" +ts=2024-04-17T09:52:46.36401476Z caller=http.go:194 level=debug traceID=46ecede10c2603f5 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.392645ms" +ts=2024-04-17T09:52:46.363873777Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 805.376µs" +ts=2024-04-17T09:52:46.363974185Z caller=http.go:194 level=debug traceID=1b48f5156a61ca69 msg="GET /debug/pprof/delta_mutex (200) 1.161082ms" +ts=2024-04-17T09:52:46.363789152Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 764.051µs" +ts=2024-04-17T09:52:46.362529898Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 790.475µs" +ts=2024-04-17T09:52:46.364031457Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 476.579µs" +ts=2024-04-17T09:52:46.361323896Z caller=http.go:194 level=debug traceID=599c62a3ac7de78a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.331755ms" +ts=2024-04-17T09:52:46.363943545Z caller=http.go:194 level=debug traceID=4da0ab0295ab6c16 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 582.609µs" +ts=2024-04-17T09:52:46.363907661Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 844.336µs" +ts=2024-04-17T09:52:46.363979594Z caller=http.go:194 level=debug traceID=6ff6d0cc340ca6e7 orgID=32133 msg="POST /ingester.v1.IngesterService/Push (200) 1.315545ms" +ts=2024-04-17T09:52:46.363939502Z caller=http.go:194 level=debug traceID=34467dc44e314b02 orgID=325123 msg="POST /ingester.v1.IngesterService/Push (200) 758.566µs" +ts=2024-04-17T09:52:46.363924229Z caller=http.go:194 level=debug traceID=1ee220e024eb5b0a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 443.231µs" +ts=2024-04-17T09:52:46.363882565Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 746.182µs" +ts=2024-04-17T09:52:46.363850696Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 487.255µs" +ts=2024-04-17T09:52:46.363769122Z caller=http.go:194 level=debug traceID=04ece9ea52939418 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 14.805487ms" +ts=2024-04-17T09:52:46.363816282Z caller=http.go:194 level=debug traceID=0052911cab18a35d orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 4.99224ms" +ts=2024-04-17T09:52:46.363754816Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 931.934µs" +ts=2024-04-17T09:52:46.363705611Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 744.005µs" +ts=2024-04-17T09:52:46.36377788Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 956.91µs" +ts=2024-04-17T09:52:46.363732729Z caller=http.go:194 level=debug traceID=6e6e579dac547044 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.036865ms" +ts=2024-04-17T09:52:46.36376503Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 851.996µs" +ts=2024-04-17T09:52:46.36375519Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 845.405µs" +ts=2024-04-17T09:52:46.363674007Z caller=http.go:194 level=debug traceID=0b0e675689e15909 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.177299ms" +ts=2024-04-17T09:52:46.363672895Z caller=http.go:194 level=debug traceID=554713b69cd1473a orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.611307ms" +ts=2024-04-17T09:52:46.36371749Z caller=http.go:194 level=debug traceID=70121da20778353f orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 846.904µs" +ts=2024-04-17T09:52:46.363695016Z caller=http.go:194 level=debug traceID=126a4a4108de5ce6 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 529.001µs" +ts=2024-04-17T09:52:46.363644321Z caller=http.go:194 level=debug traceID=6141b48c238b3d5e orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 440.298µs" +ts=2024-04-17T09:52:46.363603421Z caller=http.go:194 level=debug traceID=748df0ddf0b0f22d orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.562106ms" +ts=2024-04-17T09:52:46.363685875Z caller=http.go:194 level=debug traceID=71aeed5818159731 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 701.457µs" +ts=2024-04-17T09:52:46.363659588Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 729.084µs" +ts=2024-04-17T09:52:46.363561079Z caller=http.go:194 level=debug traceID=635516613bdc1d25 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 870.913µs" +ts=2024-04-17T09:52:46.363625435Z caller=http.go:194 level=debug traceID=2f549a41e3a5eb00 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 560.428µs" +ts=2024-04-17T09:52:46.36352246Z caller=http.go:194 level=debug traceID=0650b8c486d05d36 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.104667ms" +ts=2024-04-17T09:52:46.363534538Z caller=http.go:194 level=debug traceID=5269eebfe73cfedb orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 1.110652ms" +ts=2024-04-17T09:52:46.363579783Z caller=http.go:194 level=debug traceID=64b8a1a256934690 orgID=12372 msg="POST /ingester.v1.IngesterService/Push (200) 3.154879ms" \ No newline at end of file diff --git a/pkg/pattern/ingester_querier.go b/pkg/pattern/ingester_querier.go index 9aa8cc27c6e3..bfbaeb92aedb 100644 --- a/pkg/pattern/ingester_querier.go +++ b/pkg/pattern/ingester_querier.go @@ -67,7 +67,7 @@ func (q *IngesterQuerier) Patterns(ctx context.Context, req *logproto.QueryPatte } func prunePatterns(resp *logproto.QueryPatternsResponse, minClusterSize int) *logproto.QueryPatternsResponse { - d := drain.New(drainConfig) + d := drain.New(drain.DefaultConfig()) for _, p := range resp.Series { d.TrainPattern(p.Pattern, p.Samples) } diff --git a/pkg/pattern/stream.go b/pkg/pattern/stream.go index 2327d92fb329..036ad2d65dc0 100644 --- a/pkg/pattern/stream.go +++ b/pkg/pattern/stream.go @@ -13,49 +13,6 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -// TODO(kolesnikovae): -// -// This is crucial for Drain to ensure that the first LogClusterDepth tokens -// are constant (see https://jiemingzhu.github.io/pub/pjhe_icws2017.pdf). -// We should remove any variables such as timestamps, IDs, IPs, counters, etc. -// from these tokens. -// -// Moreover, Drain is not designed for structured logs. Therefore, we should -// handle logfmt (and, probably, JSON) logs in a special way: -// -// The parse tree should have a fixed length, and the depth should be -// determined by the number of fields in the logfmt message. -// A parsing tree should be maintained for each unique field set. - -var drainConfig = &drain.Config{ - // At training, if at the depth of LogClusterDepth there is a cluster with - // similarity coefficient greater that SimTh, then the log message is added - // to that cluster. Otherwise, a new cluster is created. - // - // LogClusterDepth should be equal to the number of constant tokens from - // the beginning of the message that likely determine the message contents. - // - // > In this step, Drain traverses from a 1-st layer node, which - // > is searched in step 2, to a leaf node. This step is based on - // > the assumption that tokens in the beginning positions of a log - // > message are more likely to be constants. Specifically, Drain - // > selects the next internal node by the tokens in the beginning - // > positions of the log message - LogClusterDepth: 8, - // SimTh is basically a ratio of matching/total in the cluster. - // Cluster tokens: "foo <*> bar fred" - // Log line: "foo bar baz qux" - // * * * x - // Similarity of these sequences is 0.75 (the distance) - // Both SimTh and MaxClusterDepth impact branching factor: the greater - // MaxClusterDepth and SimTh, the less the chance that there will be - // "similar" clusters, but the greater the footprint. - SimTh: 0.3, - MaxChildren: 100, - ParamString: "<_>", - MaxClusters: 300, -} - type stream struct { fp model.Fingerprint labels labels.Labels @@ -76,7 +33,7 @@ func newStream( labels: labels, labelsString: labels.String(), labelHash: labels.Hash(), - patterns: drain.New(drainConfig), + patterns: drain.New(drain.DefaultConfig()), }, nil }