diff --git a/docs/getting-started/alerts.yaml b/docs/getting-started/alerts.yaml
index cddd1f57da..bfb33b0345 100644
--- a/docs/getting-started/alerts.yaml
+++ b/docs/getting-started/alerts.yaml
@@ -245,7 +245,7 @@ groups:
message: |
The number of in-memory series per ingester in {{ $labels.cluster }}/{{ $labels.namespace }} is too high.
expr: |
- avg by (cluster, namespace) (cortex_ingester_memory_series) > 1.6e6
+ avg by (cluster, namespace) (cortex_ingester_memory_series) > 3.2e6
for: 2h
labels:
severity: warning
@@ -254,7 +254,7 @@ groups:
message: |
Ingesters in {{ $labels.cluster }}/{{ $labels.namespace }} ingest too many samples per second.
expr: |
- avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3
+ avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 160e3
for: 15m
labels:
severity: warning
diff --git a/docs/getting-started/dashboards/alertmanager.json b/docs/getting-started/dashboards/alertmanager.json
index b9bf3cd133..3d61215caa 100644
--- a/docs/getting-started/dashboards/alertmanager.json
+++ b/docs/getting-started/dashboards/alertmanager.json
@@ -27,232 +27,151 @@
"height": "100px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Total Alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Total Silences",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "max(cortex_alertmanager_tenants_discovered{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
}
],
"repeat": null,
@@ -267,90 +186,59 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 4,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 12,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{job=~\"$job\"})\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "APS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -365,253 +253,177 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 5,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"})\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "NPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 6,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{job=~\"$job\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"}) by(integration)\n) > 0\nor on () vector(0)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success - {{ integration }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"}) by(integration)",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed - {{ integration }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "NPS by integration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 7,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -650,157 +462,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 9,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -833,140 +745,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 11,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -987,81 +862,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 12,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1082,81 +943,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 13,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1189,81 +1036,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 14,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1284,81 +1117,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 15,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1379,81 +1198,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 16,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1474,81 +1279,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 17,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1581,235 +1372,157 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "max by(pod) (cortex_alertmanager_tenants_owned{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod Tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod Alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod Silences",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1824,244 +1537,155 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 21,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_alertmanager_sync_configs_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Syncs/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 22,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{reason}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Syncs/sec (By Reason)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 23,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "errors",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Ring Check Errors/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2076,158 +1700,115 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 24,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "{{outcome}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Initial syncs /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 25,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Initial sync duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2248,90 +1829,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 26,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "1m",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Fetch state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2346,262 +1896,169 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 27,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{job=~\"$job\"})\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Replicate state to other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 28,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{job=~\"$job\"})\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Merge state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 29,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_alertmanager_state_persist_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Persist state to remote storage /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2625,7 +2082,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-compactor-resources.json b/docs/getting-started/dashboards/cortex-compactor-resources.json
index dfaf7c870b..6c36a6b597 100644
--- a/docs/getting-started/dashboards/cortex-compactor-resources.json
+++ b/docs/getting-started/dashboards/cortex-compactor-resources.json
@@ -27,30 +27,40 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"seriesOverrides": [
{
"alias": "limit",
@@ -58,89 +68,64 @@
"fill": 0
}
],
- "spaceLength": 10,
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{job=~\"$job\",container=~\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "min(container_spec_cpu_quota{job=~\"$job\",container=~\"compactor\"} / container_spec_cpu_period{job=~\"$job\",container=~\"compactor\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "limit",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "CPU",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 2,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"seriesOverrides": [
{
"alias": "limit",
@@ -148,45 +133,28 @@
"fill": 0
}
],
- "spaceLength": 10,
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "max by(pod) (container_memory_working_set_bytes{job=~\"$job\",container=~\"compactor\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "min(container_spec_memory_limit_bytes{job=~\"$job\",container=~\"compactor\"} > 0)",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "limit",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Memory (workingset)",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "bytes",
@@ -207,61 +175,55 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 3,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (go_memstats_heap_inuse_bytes{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Memory (go heap inuse)",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "bytes",
@@ -294,63 +256,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (rate(container_network_receive_bytes_total{job=~\"$job\",pod=~\"compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Receive Bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "Bps",
@@ -371,63 +325,55 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (rate(container_network_transmit_bytes_total{job=~\"$job\",pod=~\"compactor.*\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Transmit Bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "Bps",
@@ -460,63 +406,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n job=~\"$job\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Disk Writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "Bps",
@@ -537,63 +475,55 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n job=~\"$job\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}} - {{device}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Disk Reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "Bps",
@@ -614,63 +544,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 8,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{job=~\"$job\"} /\n kubelet_volume_stats_capacity_bytes{job=~\"$job\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n job=~\"$job\",\n label_name=\"compactor\"\n }\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Disk Space Utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -712,7 +631,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-compactor.json b/docs/getting-started/dashboards/cortex-compactor.json
index f1ae6789b9..9dfb797e2f 100644
--- a/docs/getting-started/dashboards/cortex-compactor.json
+++ b/docs/getting-started/dashboards/cortex-compactor.json
@@ -33,158 +33,120 @@
"started": "#34CCEB"
},
"bars": true,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
"lines": false,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_compactor_runs_started_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "started",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_compactor_runs_completed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "completed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_compactor_runs_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per-instance runs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\nReset to 0 after the compaction run is completed for all tenants in the shard.\n\n",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 2,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "(\n cortex_compactor_tenants_processing_succeeded{job=~\"$job\"} +\n cortex_compactor_tenants_processing_failed{job=~\"$job\"} +\n cortex_compactor_tenants_skipped{job=~\"$job\"}\n) / cortex_compactor_tenants_discovered{job=~\"$job\"}\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Tenants compaction progress",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -217,160 +179,117 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Compacted blocks / sec\nRate of blocks that are generated as a result of a compaction operation.\n\n",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 3,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(prometheus_tsdb_compactions_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "blocks",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Compacted blocks / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Per-block compaction duration\nDisplay the amount of time that it has taken to generate a single compacted block.\n\n",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 4,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per-block compaction duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -403,159 +322,101 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 5,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "avg(max by(user) (cortex_bucket_blocks_count{job=~\"$job\"}))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "avg",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Average blocks / tenant",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 6,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "topk(10, max by(user) (cortex_bucket_blocks_count{job=~\"$job\"}))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{user}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Tenants with largest number of blocks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -570,152 +431,114 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 7,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "blocks",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks marked for deletion / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_compactor_blocks_cleaned_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks deletions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -752,71 +575,62 @@
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_compactor_meta_syncs_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_compactor_meta_sync_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Metadata Syncs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -837,81 +651,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 10,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Metadata Sync Duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -944,140 +744,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 12,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -1098,81 +861,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 13,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1193,81 +942,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 14,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1300,81 +1035,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 15,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1395,81 +1116,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 16,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1490,81 +1197,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 17,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1585,81 +1278,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 18,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1701,7 +1380,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-config.json b/docs/getting-started/dashboards/cortex-config.json
index 20026709e7..8592fc7f25 100644
--- a/docs/getting-started/dashboards/cortex-config.json
+++ b/docs/getting-started/dashboards/cortex-config.json
@@ -27,81 +27,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "instances"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 12,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "count(cortex_config_hash{job=~\"$job\"}) by (sha256)",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "sha256:{{sha256}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Startup config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "instances",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -116,81 +90,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "instances"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 12,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "count(cortex_runtime_config_hash{job=~\"$job\"}) by (sha256)",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "sha256:{{sha256}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Runtime config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "instances",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -214,7 +162,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-object-store.json b/docs/getting-started/dashboards/cortex-object-store.json
index 20d96b2650..aca2a08207 100644
--- a/docs/getting-started/dashboards/cortex-object-store.json
+++ b/docs/getting-started/dashboards/cortex-object-store.json
@@ -27,158 +27,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{component}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "RPS / component",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 2,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{component}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate / component",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -193,158 +138,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "RPS / operation",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 4,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate / operation",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -359,81 +249,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 5,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -454,81 +330,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 6,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -549,81 +411,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 7,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -656,81 +504,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 8,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -751,81 +585,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 9,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -846,81 +666,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 10,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -962,7 +768,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-queries.json b/docs/getting-started/dashboards/cortex-queries.json
index d2a38c2248..63cb39ee33 100644
--- a/docs/getting-started/dashboards/cortex-queries.json
+++ b/docs/getting-started/dashboards/cortex-queries.json
@@ -27,81 +27,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Queue Duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -122,84 +108,70 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_query_frontend_retries_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Retries",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -217,81 +189,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "cortex_query_frontend_queue_length{job=~\"$job\"}",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{cluster}} / {{namespace}} / {{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Queue Length",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -306,81 +249,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Queue Duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -401,81 +330,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "cortex_query_scheduler_queue_length{job=~\"$job\"}",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{cluster}} / {{namespace}} / {{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Queue Length",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -490,141 +390,101 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Intervals per Query\nThe average number of splitted queries (partitioned by time) executed a single input query.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_frontend_split_queries_total{job=~\"$job\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{job=~\"$job\", method=\"split_by_interval\"}[1m]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "splitting rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Intervals per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_cache_hits{name=~\"frontend.+\", job=~\"$job\"}[1m])) / sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", job=~\"$job\"}[1m])) or\nsum(rate(cortex_cache_hits_total{name=~\"frontend.+\", job=~\"$job\"}[1m])) / sum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", job=~\"$job\"}[1m]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Hit Rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Results Cache Hit %",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -645,81 +505,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", job=~\"$job\"}[1m])) - sum(rate(cortex_cache_hits{name=~\"frontend.+\", job=~\"$job\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", job=~\"$job\"}[1m])) - sum(rate(cortex_cache_hits_total{name=~\"frontend.+\", job=~\"$job\"}[1m]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Miss Rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Results Cache misses",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -734,64 +565,53 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Sharded Queries Ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel takes in account only type of queries which are supported by query sharding (eg. range queries).\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{job=~\"$job\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "sharded queries ratio",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Sharded Queries Ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -812,85 +632,71 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Number of Sharded Queries per Query\nHow many sharded queries have been executed for a single input query. It tracks only queries which have\nbeen successfully rewritten in a shardable way.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Number of Sharded Queries per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -920,235 +726,151 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\"}) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{slice}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Stages",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_cache_fetched_keys{job=~\"$job\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{job=~\"$job\",name=\"chunksmemcache\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{job=~\"$job\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits_total{job=~\"$job\",name=\"chunksmemcache\"}[1m]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Hit rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Chunk cache misses",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_cache_corrupt_chunks_total{job=~\"$job\"}[1m]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Corrupt chunks",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Chunk cache corruptions",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1163,81 +885,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{job=~\"$job\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Series per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1255,81 +963,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1 * sum(cluster_job:cortex_ingester_queried_chunks_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_chunks_count:sum_rate{job=~\"$job\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Chunks per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1347,81 +1041,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{job=~\"$job\"})) * 1",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{job=~\"$job\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Samples per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1451,84 +1131,70 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Number of store-gateways hit per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1546,84 +1212,70 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Refetches of missing blocks per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1641,63 +1293,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{job=~\"$job\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{job=~\"$job\"}[1m]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Failure Rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -1730,265 +1371,191 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "max(cortex_bucket_index_loaded{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Max",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "min(cortex_bucket_index_loaded{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Min",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "avg(cortex_bucket_index_loaded{job=~\"$job\"})",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Bucket indexes loaded (per querier)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_index_loads_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_bucket_index_load_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Bucket indexes load / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Bucket indexes load latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2021,235 +1588,154 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "blocks",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks queried / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_fetched_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{data_type}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Data fetched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_touched_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{data_type}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Data touched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2264,81 +1750,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Series fetch duration (per request)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2359,81 +1831,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Series merge duration (per request)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2454,81 +1912,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_store_series_result_series_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) / sum(rate(cortex_bucket_store_series_result_series_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "avg series returned",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Series returned (per request)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2543,259 +1972,176 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cortex_bucket_store_blocks_loaded{component=\"store-gateway\",job=~\"$job\"}) without (user)",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks currently loaded",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks loaded / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Blocks dropped / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2810,158 +2156,115 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "cortex_bucket_store_indexheader_lazy_load_total{job=~\"$job\"} - cortex_bucket_store_indexheader_lazy_unload_total{job=~\"$job\"}",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{pod}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Lazy loaded index-headers",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Index-header lazy load duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2982,63 +2285,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{job=~\"$job\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "hit ratio",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Series hash cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -3080,7 +2372,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-reads.json b/docs/getting-started/dashboards/cortex-reads.json
index a02f9c7aa0..558982943f 100644
--- a/docs/getting-started/dashboards/cortex-reads.json
+++ b/docs/getting-started/dashboards/cortex-reads.json
@@ -50,158 +50,104 @@
"height": "100px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the /prometheus API as\nwell as queries from the ruler.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "reqps",
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n job=~\"$job\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n) +\nsum(\n rate(\n cortex_prometheus_rule_evaluations_total{\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Instant queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Range queries per second\nRate of range queries per second being made to\nCortex via the /prometheus API.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "reqps",
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n job=~\"$job\",\n route=~\"(prometheus|api_prom)_api_v1_query_range\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Range queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
}
],
"repeat": null,
@@ -222,157 +168,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 5,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -393,73 +439,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 6,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -480,157 +512,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 8,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -651,73 +783,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 9,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -749,160 +867,260 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 12,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency (Time in Queue)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -941,157 +1159,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 14,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{job=~\"$job\", method=~\"frontend.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{job=~\"$job\", method=~\"frontend.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate{job=~\"$job\", method=~\"frontend.+\"}) / sum(cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate{job=~\"$job\", method=~\"frontend.+\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1130,157 +1448,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 16,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1301,73 +1719,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 17,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
- "format": "time_series",
- "interval": "15s",
- "intervalFactor": 2,
- "legendFormat": "",
- "legendLink": null,
- "step": 10
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
}
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per pod p99 Latency",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
},
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
+ "span": 4,
+ "targets": [
+ {
+ "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
+ "format": "time_series",
+ "interval": "15s",
+ "legendFormat": "",
+ "legendLink": null
}
- ]
+ ],
+ "title": "Per pod p99 Latency",
+ "tooltip": {
+ "sort": 2
+ },
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1388,157 +1792,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 19,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1559,73 +2063,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 20,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1646,157 +2136,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 22,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1817,73 +2407,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 23,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1898,158 +2474,118 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 25,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2070,64 +2606,53 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Hit Ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because Cortex by default has an\nin-memory blocks index cache.\n\n",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 26,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{item_type}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -2160,63 +2685,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -2237,81 +2754,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 28,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2332,63 +2835,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 29,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "items",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -2421,63 +2913,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -2498,81 +2982,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 31,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ ],
+ "title": "Latency (getmulti)",
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2593,63 +3063,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 32,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "items",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -2682,63 +3141,55 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -2759,81 +3210,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 34,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2854,63 +3291,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 35,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "items",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -2943,140 +3369,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 37,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 37,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -3097,81 +3486,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 38,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 38,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3192,81 +3567,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 39,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 39,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3299,81 +3660,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 40,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 40,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3394,81 +3741,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 41,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 41,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3489,81 +3822,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 42,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 42,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3584,81 +3903,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 43,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 43,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3691,140 +3996,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 44,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"querier\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 45,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 45,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"querier\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -3845,81 +4113,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 46,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 46,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -3940,81 +4194,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 47,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 47,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -4047,81 +4287,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 48,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 48,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -4142,81 +4368,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 49,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 49,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -4237,81 +4449,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 50,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 50,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -4332,81 +4530,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
- "id": 51,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
},
- "lines": true,
- "linewidth": 1,
+ "id": 51,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -4448,7 +4632,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-rollout-progress.json b/docs/getting-started/dashboards/cortex-rollout-progress.json
index fe24526292..9bd20e9e95 100644
--- a/docs/getting-started/dashboards/cortex-rollout-progress.json
+++ b/docs/getting-started/dashboards/cortex-rollout-progress.json
@@ -22,10 +22,6 @@
],
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -55,7 +51,6 @@
"unit": "percentunit"
}
},
- "fill": 1,
"gridPos": {
"h": 8,
"w": 10,
@@ -63,19 +58,7 @@
"y": 0
},
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
"options": {
"displayMode": "basic",
"orientation": "horizontal",
@@ -87,15 +70,6 @@
"values": false
}
},
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{job=~\"$job\",statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{job=~\"$job\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{job=~\"$job\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n",
@@ -114,47 +88,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Rollout progress",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "bargauge",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "bargauge"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -176,7 +113,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -184,28 +120,16 @@
"y": 0
},
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n",
@@ -218,47 +142,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Writes - 2xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -288,7 +175,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -296,28 +182,16 @@
"y": 0
},
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n",
@@ -330,47 +204,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Writes - 4xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -396,7 +233,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -404,28 +240,16 @@
"y": 0
},
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n",
@@ -438,47 +262,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Writes - 5xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -508,7 +295,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 8,
@@ -516,28 +302,16 @@
"y": 0
},
"id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}))\n",
@@ -550,47 +324,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Writes 99th Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "title": "Writes 99th latency",
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -612,7 +349,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -620,28 +356,16 @@
"y": 4
},
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
@@ -654,47 +378,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Reads - 2xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -724,7 +411,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -732,28 +418,16 @@
"y": 4
},
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
@@ -766,47 +440,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Reads - 4xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -832,7 +469,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 2,
@@ -840,28 +476,16 @@
"y": 4
},
"id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
@@ -874,47 +498,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Reads - 5xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -944,7 +531,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 4,
"w": 8,
@@ -952,28 +538,16 @@
"y": 4
},
"id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n",
@@ -986,47 +560,10 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Reads 99th Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "title": "Reads 99th latency",
+ "type": "stat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
@@ -1056,7 +593,6 @@
},
"overrides": [ ]
},
- "fill": 1,
"gridPos": {
"h": 8,
"w": 10,
@@ -1064,34 +600,13 @@
"y": 8
},
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
"options": {
"text": {
"titleSize": 14,
"valueSize": 14
}
},
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "kube_deployment_status_replicas_unavailable{job=~\"$job\", deployment=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"}\n> 0\n",
@@ -1114,41 +629,8 @@
"step": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Unhealthy pods",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "stat"
},
{
"datasource": "$datasource",
@@ -1214,12 +696,29 @@
"type": "table"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"gridPos": {
"h": 8,
"w": 8,
@@ -1227,65 +726,34 @@
"y": 8
},
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"targets": [
{
"expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}))[1h:])\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "writes",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "reads",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency vs 24h ago",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -1321,7 +789,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-scaling.json b/docs/getting-started/dashboards/cortex-scaling.json
index f481f25bdd..fcadcc3bd1 100644
--- a/docs/getting-started/dashboards/cortex-scaling.json
+++ b/docs/getting-started/dashboards/cortex-scaling.json
@@ -49,12 +49,27 @@
"height": "400px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
"overrides": [
{
"matcher": {
@@ -142,81 +157,34 @@
}
]
},
- "fill": 1,
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"sort": {
"col": 0,
"desc": false
},
- "spaceLength": 10,
"span": 12,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sort_desc(\n cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n > ignoring(reason) group_left\n cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n",
"format": "table",
"instant": true,
- "intervalFactor": 2,
"legendFormat": "",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Workload-based scaling",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
"transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "table"
}
],
"repeat": null,
@@ -240,7 +208,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-slow-queries.json b/docs/getting-started/dashboards/cortex-slow-queries.json
index cf1ac17c49..d2853710a9 100644
--- a/docs/getting-started/dashboards/cortex-slow-queries.json
+++ b/docs/getting-started/dashboards/cortex-slow-queries.json
@@ -171,7 +171,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/cortex-writes.json b/docs/getting-started/dashboards/cortex-writes.json
index f7b3070e9d..296a65a655 100644
--- a/docs/getting-started/dashboards/cortex-writes.json
+++ b/docs/getting-started/dashboards/cortex-writes.json
@@ -50,308 +50,200 @@
"height": "100px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Samples / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cortex_ingester_memory_series{job=~\"$job\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{job=~\"$job\"}))\n",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Active Series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "count(count by(user) (cortex_ingester_active_series{job=~\"$job\"}))",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "reqps",
"id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[5m]))",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
}
],
"repeat": null,
@@ -372,157 +264,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -543,73 +535,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 8,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -630,157 +608,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -801,73 +879,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 11,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -888,157 +952,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{job=~\"$job\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1077,157 +1241,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
- "format": "time_series",
- "interval": "15s",
- "intervalFactor": 2,
- "legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
+ "linewidth": 0,
+ "links": [ ],
+ "options": {
+ "legend": {
+ "showLegend": true
},
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "span": 4,
+ "stack": true,
+ "targets": [
{
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
+ "format": "time_series",
+ "interval": "15s",
+ "legendFormat": "{{status}}",
+ "refId": "A"
}
- ]
+ ],
+ "title": "Requests / sec",
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1248,73 +1512,59 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"fill": 0,
"id": 16,
"legend": {
"show": false
},
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Per pod p99 Latency",
"tooltip": {
"sort": 2
},
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1335,157 +1585,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{job=~\"$job\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1522,168 +1872,127 @@
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_shipper_uploads_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Uploaded blocks / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Upload latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1720,168 +2029,127 @@
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_tsdb_compactions_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Compactions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Compactions latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1918,329 +2186,237 @@
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "WAL truncations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"failed": "#E24D42",
"successful": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "successful",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Checkpoints created / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
"description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "avg",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "WAL truncations latency (includes checkpointing)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
"aliasColors": {
"WAL": "#E24D42",
"mmap-ed chunks": "#E28A42"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_wal_corruptions_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "WAL",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "mmap-ed chunks",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Corruptions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ops",
@@ -2282,7 +2458,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/dashboards/ruler.json b/docs/getting-started/dashboards/ruler.json
index d17fcad627..e48964da33 100644
--- a/docs/getting-started/dashboards/ruler.json
+++ b/docs/getting-started/dashboards/ruler.json
@@ -27,308 +27,200 @@
"height": "100px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cortex_ruler_managers_total{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Active Configurations",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "short",
"id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(cortex_prometheus_rule_group_rules{job=~\"$job\"})",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Total Rules",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "reqps",
"id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[5m]))",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Read from Ingesters - QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"format": "reqps",
"id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[5m]))",
"format": "time_series",
"instant": true,
- "intervalFactor": 2,
"refId": "A"
}
],
"thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
"title": "Write to Ingesters - QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "singlestat"
}
],
"repeat": null,
@@ -343,167 +235,107 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_prometheus_rule_evaluations_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "success",
- "legendLink": null,
- "step": 10
+ "legendLink": null
},
{
"expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "failed",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "EPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{job=~\"$job\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "average",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -524,157 +356,257 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "legendFormat": "99th percentile",
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3",
"format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "legendFormat": "50th percentile",
+ "refId": "B"
},
{
"expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})",
"format": "time_series",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -695,81 +627,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (route, le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ route }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per route p99 Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "title": "Per route p99 latency",
+ "type": "timeseries"
}
],
"repeat": null,
@@ -790,160 +693,260 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -982,160 +985,260 @@
"3xx": "#6ED0E0",
"4xx": "#EF843C",
"5xx": "#E24D42",
+ "OK": "#7EB26D",
+ "cancel": "#A9A9A9",
"error": "#E24D42",
"success": "#7EB26D"
},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 100,
+ "lineWidth": 0,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ }
+ },
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "1xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EAB839",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "2xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "3xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#6ED0E0",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "4xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#EF843C",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "5xx"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "OK"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "cancel"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#A9A9A9",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "error"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#E24D42",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "success"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "#7EB26D",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
+ },
"fill": 10,
"id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
"stack": true,
- "steppedLine": false,
"targets": [
{
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n",
+ "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{status}}",
- "refId": "A",
- "step": 10
+ "refId": "A"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": [ ]
+ },
"id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 6,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -1168,84 +1271,70 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Number of store-gateways hit per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1263,84 +1352,70 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{job=~\"$job\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Refetches of missing blocks per Query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
- "format": "short",
+ "format": "ms",
"label": null,
"logBase": 1,
"max": null,
@@ -1358,63 +1433,52 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{job=~\"$job\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{job=~\"$job\"}[1m]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Failure Rate",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -1447,235 +1511,148 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{job=~\"$job\"}[$__rate_interval]))\n> 0\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Delivery Errors",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{job=~\"$job\"}[$__rate_interval])) > 0\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Queue Length",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{job=~\"$job\"}[$__rate_interval])) > 0\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Dropped",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1690,235 +1667,148 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{job=~\"$job\"}[$__rate_interval])) > 0",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Missed Iterations",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 4,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval])) > 0",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ rule_group }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Failures",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -1933,81 +1823,52 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [ ]
+ },
"id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 12,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))\n",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{ user }}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
}
],
"repeat": null,
@@ -2022,140 +1883,103 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": [ ]
+ },
"fill": 10,
"id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
"linewidth": 0,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
"stack": true,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "rps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
+ "type": "timeseries"
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": [ ]
+ },
"id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "{{operation}}",
- "legendLink": null,
- "step": 10
+ "legendLink": null
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Error rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "percentunit",
@@ -2176,81 +2000,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2271,81 +2081,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2378,81 +2174,67 @@
"height": "250px",
"panels": [
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2473,81 +2255,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2568,81 +2336,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2663,81 +2417,67 @@
]
},
{
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
"datasource": "$datasource",
- "fill": 1,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 1,
+ "lineWidth": 1,
+ "pointSize": 5,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [ ]
+ },
+ "unit": "short"
+ },
+ "overrides": [ ]
+ },
"id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
"links": [ ],
"nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
+ "options": {
+ "legend": {
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
"span": 3,
- "stack": false,
- "steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
+ "refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
+ "refId": "B"
},
{
"expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))",
"format": "time_series",
"interval": "15s",
- "intervalFactor": 2,
"legendFormat": "Average",
- "refId": "C",
- "step": 10
+ "refId": "C"
}
],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
"title": "Latency of Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
+ "type": "timeseries",
"yaxes": [
{
"format": "ms",
@@ -2779,7 +2519,7 @@
"value": "Cortex"
},
"hide": 0,
- "label": null,
+ "label": "Data source",
"name": "datasource",
"options": [ ],
"query": "prometheus",
diff --git a/docs/getting-started/rules.yaml b/docs/getting-started/rules.yaml
index 6a5646a769..b458ea8688 100644
--- a/docs/getting-started/rules.yaml
+++ b/docs/getting-started/rules.yaml
@@ -364,7 +364,7 @@ groups:
cortex_ingester_memory_series
)[24h:]
)
- / 1500000
+ / 3000000
)
labels:
deployment: ingester
@@ -373,7 +373,7 @@ groups:
- expr: |
ceil(
sum by (cluster, namespace) (cortex_overrides{limit_name="max_global_series_per_user"})
- * 3 * 0.59999999999999998 / 1500000
+ * 3 * 0.59999999999999998 / 3000000
)
labels:
deployment: ingester