just found a new issue :) Looks like query results are duplicated (in some cases?). Here's a query which cloki generated:
WITH
str_sel AS
(
SELECT DISTINCT fingerprint
FROM cloki.time_series
WHERE (JSONHas(labels, 'pod_namespace') = 1) AND (JSONExtractString(labels, 'pod_namespace') = 'impression-notifier-staging')
),
sel_a AS
(
SELECT
time_series.labels AS labels,
samples.string AS string,
samples.fingerprint AS fingerprint,
samples.timestamp_ns AS timestamp_ns
FROM cloki.samples_v3 AS samples
LEFT JOIN cloki.time_series AS time_series ON samples.fingerprint = time_series.fingerprint
WHERE ((samples.timestamp_ns >= 1649852884000000000) AND (samples.timestamp_ns <= 1649856485000000000)) AND (samples.fingerprint IN (
SELECT fingerprint
FROM str_sel
)) AND (extractAllGroups(string, '(16E5768DBDF88309-53C83D0EA34C4CFA)') != []) AND (extractAllGroups(string, '(32000)') != [])
ORDER BY
timestamp_ns DESC,
labels DESC
LIMIT 1000
)
SELECT *
FROM sel_a
ORDER BY
labels DESC,
timestamp_ns DESC
FORMAT JSONEachRow
Query id: 53729179-df41-459c-b711-2c565f3b8b25
{"labels":"{\"container_image\":\"...\",\"container_name\":\"impression-notifier\",\"level\":\"debug\",\"pod_namespace\":\"impression-notifier-staging\",\"pod_node_name\":\"kube38-63\",\"stream\":\"stderr\"}","string":"{\"container_id\":\"containerd:\/\/d133fc0303678286467a9643e08993801ccd4558c7df4cac48f52257c8a9184a\",\"file\":\"\/var\/log\/pods\/impression-notifier-staging_impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt_dcc24d17-637f-4f8d-a079-372e24273c8b\/impression-notifier\/0.log\",\"msg\":\"decoded report\",\"pod_ip\":\"...\",\"pod_ips\":[\"...\"],\"pod_labels\":{\"cdk8s.deployment\":\"impression-notifier-staging-Deployment-c888ef57\",\"pod-template-hash\":\"78d64b656c\"},\"pod_name\":\"impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt\",\"pod_uid\":\"dcc24d17-637f-4f8d-a079-372e24273c8b\",\"report\":{\"duration_ms\":4000,\"event_id\":\"47193\",\"session_id\":\"16E5768DBDF88309-53C83D0EA34C4CFA\",\"start_ms\":32000},\"source_type\":\"kubernetes_logs\",\"time\":\"2022-04-13T13:07:07Z\"}","fingerprint":"6434154905068867235","timestamp_ns":"1649855227871304633"}
{"labels":"{\"container_image\":\"...\",\"container_name\":\"impression-notifier\",\"level\":\"debug\",\"pod_namespace\":\"impression-notifier-staging\",\"pod_node_name\":\"kube38-63\",\"stream\":\"stderr\"}","string":"{\"container_id\":\"containerd:\/\/d133fc0303678286467a9643e08993801ccd4558c7df4cac48f52257c8a9184a\",\"file\":\"\/var\/log\/pods\/impression-notifier-staging_impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt_dcc24d17-637f-4f8d-a079-372e24273c8b\/impression-notifier\/0.log\",\"msg\":\"decoded report\",\"pod_ip\":\"...\",\"pod_ips\":[\"...\"],\"pod_labels\":{\"cdk8s.deployment\":\"impression-notifier-staging-Deployment-c888ef57\",\"pod-template-hash\":\"78d64b656c\"},\"pod_name\":\"impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt\",\"pod_uid\":\"dcc24d17-637f-4f8d-a079-372e24273c8b\",\"report\":{\"duration_ms\":4000,\"event_id\":\"47193\",\"session_id\":\"16E5768DBDF88309-53C83D0EA34C4CFA\",\"start_ms\":32000},\"source_type\":\"kubernetes_logs\",\"time\":\"2022-04-13T13:07:07Z\"}","fingerprint":"6434154905068867235","timestamp_ns":"1649855227871304633"}
{"labels":"{\"container_image\":\"...\",\"container_name\":\"impression-notifier\",\"level\":\"debug\",\"pod_namespace\":\"impression-notifier-staging\",\"pod_node_name\":\"kube38-63\",\"stream\":\"stderr\"}","string":"{\"container_id\":\"containerd:\/\/d133fc0303678286467a9643e08993801ccd4558c7df4cac48f52257c8a9184a\",\"file\":\"\/var\/log\/pods\/impression-notifier-staging_impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt_dcc24d17-637f-4f8d-a079-372e24273c8b\/impression-notifier\/0.log\",\"msg\":\"decoded report\",\"pod_ip\":\"...\",\"pod_ips\":[\"...\"],\"pod_labels\":{\"cdk8s.deployment\":\"impression-notifier-staging-Deployment-c888ef57\",\"pod-template-hash\":\"78d64b656c\"},\"pod_name\":\"impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt\",\"pod_uid\":\"dcc24d17-637f-4f8d-a079-372e24273c8b\",\"report\":{\"duration_ms\":4000,\"event_id\":\"47193\",\"session_id\":\"16E5768DBDF88309-53C83D0EA34C4CFA\",\"start_ms\":32000},\"source_type\":\"kubernetes_logs\",\"time\":\"2022-04-13T13:07:07Z\"}","fingerprint":"6434154905068867235","timestamp_ns":"1649855227871304633"}
3 rows in set. Elapsed: 0.192 sec. Processed 1.50 million rows, 67.88 MB (7.83 million rows/s., 353.36 MB/s.)
Here's a query I wrote which gets the same message. It is clear that there's only one row over there:
SELECT *
FROM samples_v3
WHERE (position(string, '16E5768DBDF88309-53C83D0EA34C4CFA') > 0) AND (position(string, '32000') > 0) AND (JSONExtractString(string, 'pod_name') = 'impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt')
Query id: cc911e5a-8c7a-4d9b-84f2-f9c3c46df073
┌─────────fingerprint─┬────────timestamp_ns─┬─value─┬─string─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ 6434154905068867235 │ 1649855227871304633 │ 0 │ {"container_id":"containerd://d133fc0303678286467a9643e08993801ccd4558c7df4cac48f52257c8a9184a","file":"/var/log/pods/impression-notifier-staging_impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt_dcc24d17-637f-4f8d-a079-372e24273c8b/impression-notifier/0.log","msg":"decoded report","pod_ip":"...","pod_ips":["..."],"pod_labels":{"cdk8s.deployment":"impression-notifier-staging-Deployment-c888ef57","pod-template-hash":"78d64b656c"},"pod_name":"impression-notifier-staging-deployment-c80dda91-78d64b656c4z2tt","pod_uid":"dcc24d17-637f-4f8d-a079-372e24273c8b","report":{"duration_ms":4000,"event_id":"47193","session_id":"16E5768DBDF88309-53C83D0EA34C4CFA","start_ms":32000},"source_type":"kubernetes_logs","time":"2022-04-13T13:07:07Z"} │
└─────────────────────┴─────────────────────┴───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
1 rows in set. Elapsed: 20.369 sec. Processed 359.86 million rows, 344.48 GB (17.67 million rows/s., 16.91 GB/s.)