{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "enable": true, "expr": "sum(ALERTS{job=~\"$job\", instance=~\"$instance\", alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)", "hide": false, "iconColor": "red", "name": "alerts", "titleFormat": "{{alertname}}" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "enable": true, "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(version) unless (sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"} offset $__interval) by(version))", "hide": true, "iconColor": "dark-blue", "name": "version change", "textFormat": "{{version}}", "titleFormat": "Version change" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "enable": true, "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", "hide": false, "iconColor": "dark-yellow", "name": "restarts", "textFormat": "{{job}} restarted" } ] }, "description": "Overview for cluster VictoriaMetrics v1.117.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 147, "links": [ { "icon": "doc", "tags": [], "targetBlank": true, "title": "Docs", "type": "link", "url": "https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/" }, { "asDropdown": false, "icon": "question", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Troubleshooting", "tooltip": "", "type": "link", "url": "https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#troubleshooting" }, { "icon": "info", "tags": [], "targetBlank": true, "title": "Community support", "type": "link", "url": "https://victoriametrics.com/support/community-support/" }, { "asDropdown": false, "icon": "bolt", "includeVars": false, "keepTime": false, "tags": [], "targetBlank": true, "title": "Enterprise support", "tooltip": "", "type": "link", "url": "https://victoriametrics.com/support/enterprise-support/" }, { "icon": "info", "tags": [], "targetBlank": true, "title": "Releases", "type": "link", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" } ], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 137, "panels": [], "title": "Stats", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are in storage", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 1 }, "id": 131, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total datapoints", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the datapoints ingestion rate, including replication factor.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 1 }, "id": 124, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Ingestion rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate of HTTP read requests.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "req/s" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 1 }, "id": 130, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Read requests", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total number of available CPUs for all VM components. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 1 }, "id": 126, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Available CPU", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour. High value may result in ingestion slowdown.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 4 }, "id": 34, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}[1h]))", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Active series", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total amount of used disk space", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 4 }, "id": 35, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": false, "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Disk space usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Average disk usage per datapoint.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 4 }, "id": 112, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Bytes per point", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total size of available memory for all VM components.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 18, "y": 4 }, "id": 128, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Available memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false, "minWidth": 50 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "displayName", "value": "Count" } ] } ] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 7 }, "id": 149, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Count" } ] }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)", "format": "table", "instant": true, "range": false, "refId": "A" } ], "title": "", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 16, "x": 8, "y": 7 }, "id": 62, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", "format": "time_series", "instant": false, "legendFormat": "{{job}}", "refId": "A" } ], "title": "Uptime ($job)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 10, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How many [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are inserted into cluster per second by protocol before the replication. Check vminsert metrics if there are any issues with ingestion.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 2, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (type) > 0 ", "interval": "", "legendFormat": "{{type}}", "range": true, "refId": "A" } ], "title": "Datapoints ingestion rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 6, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{path}}", "range": true, "refId": "A" } ], "title": "Requests rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series) with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown and high memory usage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 12, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "Active time series", "refId": "A" } ], "title": "Active time series ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "id": 8, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{path}}", "range": true, "refId": "A" } ], "title": "Query duration 0.99 quantile ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "id": 52, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{path}} ({{job}})", "range": true, "refId": "A" } ], "title": "Requests error rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=203&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "id": 104, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, level) > 0", "format": "time_series", "hide": false, "interval": "5m", "intervalFactor": 1, "legendFormat": "{{job}} - {{level}}", "range": true, "refId": "A" } ], "title": "Logging rate", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 46, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How much resident memory (RAM) the process is actually using, compared to its allowed container or system limit.\n\n- Good: Below 70% most of the time, maybe spiking a bit under load.\n- Bad: Above 90% for more than 5 minutes = risk of out-of-memory (OOM) kill.\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown - RSS memory usage", "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" }, { "targetBlank": true, "title": "Drilldown - Memory usage breakdown", "url": "/d/oS7Bi_0Wz?viewPanel=225&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "id": 66, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RSS memory % usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "id": 64, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CPU ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 88 }, "id": 138, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RSS anonymous memory % usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "CPU pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html). Helps troubleshoot high CPU usage or throttling.\nLower is better.\n\n- waiting: The percentage of time at least one task in the process was ready to run (runnable) but couldn't get scheduled on the CPU.\n- stalled: The percentage of time all tasks in the process (except idle ones) were unable to get CPU time — a full CPU stall.\n\nIf there's a CPU burst, it's normal to see waiting or stalled > 1%. It only becomes a concern if it consistently climbs above 5–10% and aligns with latency spikes or GC slowdowns.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 88 }, "id": 219, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_cpu_waiting_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - waiting", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_cpu_stalled_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - stalled", "range": true, "refId": "B" } ], "title": "CPU pressure", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows memory pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nLower is better.\n\n- waiting: Time fraction where at least one thread was blocked on memory.\n- stalled: Time fraction where every thread was blocked on memory (severe pressure).\n\nIf queries slow down and both series spike, the host is likely limited by RAM or I/O throughput.\n\nSee also major page faults rate panel in Troubleshooting section if this metric continued to be high.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 96 }, "id": 220, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_memory_waiting_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - waiting", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_memory_stalled_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - stalled", "range": true, "refId": "B" } ], "title": "Memory pressure", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of bytes read/write from the storage layer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read .*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 96 }, "id": 122, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{job}}", "range": true, "refId": "B" } ], "title": "Disk writes/reads ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/max.*/" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 104 }, "id": 117, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], "title": "Open FDs usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of read/write syscalls such as read, pread, write, pwrite.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read .*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 104 }, "id": 204, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(process_io_read_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(process_io_write_syscalls_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{job}}", "range": true, "refId": "B" } ], "title": "Disk write/read calls ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 112 }, "id": 68, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Goroutines ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows IO pressure based on [Pressure Stall Information](https://docs.kernel.org/accounting/psi.html).\nThe lower the better.\n\n- waiting: at least one runnable thread blocked on block-`I/O` (disk, NVMe, network-storage) while others could still make progress.\n- stalled: all non-idle threads simultaneously waiting on `I/O`; no useful user code ran during these periods → true `I/O` thrashing.\n\nIf stalled > 0 while querying, it's recommended to increase queue depth on NVMe, raise blk-mq budgets, or relax cgroup I/O limits.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 112 }, "id": 221, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_io_waiting_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - waiting", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(process_pressure_io_stalled_seconds_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{job}} - stalled", "range": true, "refId": "B" } ], "title": "IO pressure", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 120 }, "id": 70, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Threads ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 120 }, "id": 119, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", "interval": "1m", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], "title": "TCP connections ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percent of CPU spent on garbage collection.\n\nIf % is high, then CPU usage can be decreased by changing GOGC to higher values. Increasing GOGC value will increase memory usage, and decrease CPU usage.\n\nTry searching for keyword `GOGC` at https://docs.victoriametrics.com/victoriametrics/troubleshooting/ ", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 128 }, "id": 210, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n rate(go_gc_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) \n / rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n ) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CPU spent on GC ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 128 }, "id": 120, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "TCP connections rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate of allocations in memory. Sudden increase in allocations would mean increased pressure on Go Garbage Collector and can saturate CPU resources of the application.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 136 }, "id": 218, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[$__rate_interval])) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Memory allocations rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the time goroutines have spent in runnable state before actually running. The lower is better.\n\nHigh values or values exceeding the threshold is usually a sign of insufficient CPU resources or CPU throttling. \n\nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0.1 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 136 }, "id": 213, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job, instance, le))) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Go scheduling latency", "type": "timeseries" } ], "title": "Resource usage ($job)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 38 }, "id": 106, "panels": [ { "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 204 }, "id": 211, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "See [Troubleshooting](https://docs.victoriametrics.com/victoriametrics/troubleshooting/) docs.", "mode": "markdown" }, "pluginVersion": "12.2.0", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected [cardinality](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#cardinality) \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "new series over 24h" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.axisSoftMin", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 206 }, "id": 102, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "churn rate", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))", "hide": false, "interval": "", "legendFormat": "new series over 24h", "refId": "B" } ], "title": "Churn rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The percentage of [slow inserts](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-a-slow-insert) compared to the total ingestion rate. \n\nThe lower the better. \n\nIn short, slow insert is a cache miss. There are following reasons for slow inserts to go up: \n* Ingestion of completely new, not seen before time series;\n* [Re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) of series when one or more vmstorage nodes are unavailable;\n* Not enough memory to maintain big enough caches for the current workload.\n\nIf percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of [active time series](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-an-active-time-series). \n\nSee [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183) for details.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent", "value": 0 }, { "color": "red", "value": 0.1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 206 }, "id": 108, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "slow inserts", "range": true, "refId": "A" } ], "title": "Slow inserts", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Merge assist happens when vmstorage can't keep up with merging parts. This is usually a sign of overload for vmstorage.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 248 }, "id": 170, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(vm_assisted_merges_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type) > 0", "format": "time_series", "interval": "5m", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Assisted merges ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows % of slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.\n\nThe less value is better.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 248 }, "id": 107, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", "interval": "", "legendFormat": "slow queries %", "range": true, "refId": "A" } ], "title": "Slow queries % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 256 }, "id": 144, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache usage % by type ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows cache miss ratio. Lower is better.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 256 }, "id": 58, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(type)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache miss ratio ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The value is above 0 when the vmstorage at the given `addr` communicates to the given vminsert node that it cannot accept new data because it is in the read-only mode.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 265 }, "id": 142, "links": [ { "targetBlank": true, "title": "Readonly mode", "url": "https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode" } ], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(vm_rpc_vmstorage_is_read_only{job=~\"$job_insert\", instance=~\"$instance\"}) by(instance, addr) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} => {{addr}}", "range": true, "refId": "A" } ], "title": "Storage in readonly status for vminsert ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows rate of deduplicated samples during [deduplication](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) or [downsampling](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling), according to the configured `-dedup.minScrapeInterval` or `-downsampling.period`.\n\nShould be empty if `-dedup.minScrapeInterval` and `-downsampling.period` isn't set.\n\nDeduplication happens in two places:\n* `type=\"merge\"` - during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage) by vmstorages. Deduplication during merges is permanent.\n* `type=\"select\"` - during [read queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data) by vmselects. Deduplication happens in-flight and has no permanent effect.\n\nThe more samples need to be deduplicated, the higher will be resource usage on vmselects or vmstorages.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 265 }, "id": 215, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_deduplicated_samples_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, type) >0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Deduplication rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows how many samples were ignored or dropped on insertion due to various reasons:\n* timestamp out of retention period or timestamp in future;\n* invalid metric name;\n* exceeding limit for labels length or number;\n* dropped by [relabeling configuration](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#relabeling).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 273 }, "id": 135, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(vm_rows_ignored_total{job=~\"$job\", instance=~\"$instance\"}[1h])) by (reason)", "hide": false, "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(increase(vm_relabel_metrics_dropped_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", "hide": false, "instant": false, "legendFormat": "relabeling", "range": true, "refId": "B" } ], "title": "Samples dropped for last 1h ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of partial results served by `vmselects`.\n\n`vmselect` continues serving queries if at least a single vmstorage nodes is available. It marks responses as `partial` for queries served from the remaining healthy vmstorage nodes.\n\nIf you prefer consistency over availability then run vmselect nodes with `-search.denyPartialResponse` cmd-line flag. In this case vmselect returns an error if at least a single vmstorage node is unavailable.\n\nSee more at [cluster availability](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 273 }, "id": 217, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(increase(vm_partial_results_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0", "format": "time_series", "instant": false, "legendFormat": "partial results", "refId": "A" } ], "title": "Partial query results ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "footer": { "reducers": [] }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hideFrom.viz", "value": true } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 281 }, "id": 183, "options": { "cellHeight": "sm", "filterable": true, "showHeader": true, "showSearch": true, "sortBy": [ { "desc": true, "displayName": "job" } ] }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Non-default flags", "transformations": [ { "id": "groupBy", "options": { "fields": { "instance": { "aggregations": [ "uniqueValues" ], "operation": "aggregate" }, "job": { "aggregations": [], "operation": "groupby" }, "name": { "aggregations": [], "operation": "groupby" }, "value": { "aggregations": [], "operation": "groupby" } } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 281 }, "id": 214, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "instant": false, "legendFormat": "{{job}}", "refId": "A" } ], "title": "Restarts ($job)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${ds}" }, "description": "Shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters).\n\nApplying downsampling or retention filters happens in background. `vmstorage` schedules this process periodically when finds data partitions that satisfy configured retention or downsampling rules.\n\nThis process could consume additional CPU and Disk IO resources.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 289 }, "id": 222, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${ds}" }, "editorMode": "code", "expr": "sum(vm_downsampling_partitions_scheduled_size_bytes{job=~\"$job\", instance=~\"$instance\"}) > 0", "legendFormat": "downsampling", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${ds}" }, "editorMode": "code", "expr": "sum(vm_retention_partitions_scheduled_size_bytes{job=~\"$job\", instance=~\"$instance\"}) > 0", "hide": false, "instant": false, "legendFormat": "retention filters", "range": true, "refId": "B" } ], "title": "Partitions scheduled for re-processing", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${ds}" }, "description": "Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=224&var-job=${__field.labels.job}&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 289 }, "id": 223, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${ds}" }, "editorMode": "code", "expr": "sum(rate(process_major_pagefaults_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job) > 0", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Major page faults rate ($instance)", "type": "timeseries" } ], "title": "Troubleshooting", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, "id": 48, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows number of pushed and sent rows. \n* `Pushed rows` - rows added to internal inserter buffers before send\n* `Sent rows` - successfully transmitted rows to storage nodes\n\nPlease note, it could be that `Sent > Pushed` because of the replication factor.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 8371 }, "id": 76, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rpc_rows_pushed_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pushed", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rpc_rows_sent_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Sent", "range": true, "refId": "E" } ], "title": "Rows ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Rate of RPC errors by type:\n* `Connection` - the number of connection errors to vmstorage node\n* `Dial` - the number of dial errors to vmstorage node.\n* `Handshake` - the number of handshake errors to vmstorage node\n* `Rerouted` - errors appeared during rerouting of rows from un-healthy storage node to a healthy one.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 8371 }, "id": 86, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rpc_connection_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Connection", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(rate(vm_rpc_dial_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Dial", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(rate(vm_rpc_handshake_errors_total{job=~\"$job\",instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Handshake", "refId": "E" } ], "title": "RPC errors ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The number of rows rerouted to the vmstorage node from other nodes when they were unhealthy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8380 }, "id": 80, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_rpc_rows_rerouted_to_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Rows ($instance) rerouted to ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The number of rows rerouted from the vmstorage node to healthy nodes when the given node was unhealthy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8380 }, "id": 78, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(rate(vm_rpc_rows_rerouted_from_here_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{addr}}", "refId": "A" } ], "title": "Rows ($instance) rerouted from", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The number of rows or bytes that vminsert internal buffer contains at the moment.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "bytes" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8388 }, "id": 82, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(vm_rpc_buf_pending_bytes{job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "bytes", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(vm_rpc_rows_pending{job=~\"$job\", instance=~\"$instance\"})", "legendFormat": "rows", "refId": "B" } ], "title": "Pending", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Network usage by internal VictoriaMetrics RPC protocol", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8388 }, "id": 74, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) * 8", "legendFormat": "network usage", "range": true, "refId": "A" } ], "title": "RPC network usage ($instance)", "type": "timeseries" } ], "title": "Interconnection ($job)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, "id": 24, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of datapoints ingested into storage nodes per second. This metric doesn't show all stored datapoints since some of them may be dropped because of wrong timestamps or decode errors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1037 }, "id": 100, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", "format": "time_series", "intervalFactor": 1, "legendFormat": "ingestion rate", "range": true, "refId": "A" } ], "title": "Ingestion rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the approx time needed to reach 100% of allowed disk capacity for at least one vmstorage node based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1037 }, "id": 113, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "min((vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n (rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - \n sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])) without(type)) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type)\n )\n +\n rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) without(type)\n )\n) > 0)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "min ETA", "range": true, "refId": "A" } ], "title": "Storage full ETA ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1045 }, "id": 151, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "CPU usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=$job_storage&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1045 }, "id": 167, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "quantile(0.5,\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "Memory (anon) usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of ongoing Cluster Native insert API operations sent to `vmstorage`, where:\n* `max` - the maximum number of calls that can be processed in parallel;\n* `current` - Shows the number of ongoing insert operations being processed by the storage.\n\nEach successful API call results in a flush to disk. The `max` value is controlled by the `-maxConcurrentInserts` flag set on the storage (2 * cgroup.AvailableCPUs() by default). \n\nIf `current` consistently reaches `max`, it indicates that the storage is overloaded and likely requires more CPU (see CPU usage) or disks with higher IOPS (see disk read/write panels in the Resource Usage row)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1053 }, "id": 212, "links": [ { "targetBlank": true, "title": "Related discussion", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632" } ], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(vm_concurrent_insert_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "current", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", "hide": false, "interval": "", "legendFormat": "max", "range": true, "refId": "B" } ], "title": "Concurrent inserts ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of concurrently executed read requests, where:\n* `max` - equal to `-search.maxConcurrentRequest` cmd-line flag;\n* `current` - current number of concurrent select requests executed by storage.\n\nEach concurrent select request requires RAM, CPU and disk IO resources during processing. Too many concurrent requests could result into OOM exceptions.\n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).\n\nSee more at https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#resource-usage-limits", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1053 }, "id": 133, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(vm_vmselect_concurrent_requests_current{job=~\"$job_storage\", \n instance=~\"$instance\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "current", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(vm_vmselect_concurrent_requests_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", "hide": false, "interval": "", "legendFormat": "max", "range": true, "refId": "B" } ], "title": "Concurrent selects ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The max number of on-going merges across storage nodes.\n The drastic change in number of merges could be a sign of on-going deduplication/downsampling activity.\n It is expected to have high numbers for `storage/small` metric.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1094 }, "id": 54, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(max_over_time(vm_active_merges{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Active merges ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The number of rows merged per second by storage nodes. Merge speed depends on available CPU and disk IO bandwidth.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1094 }, "id": 55, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rows_merged_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(type)", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Merge speed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1102 }, "id": 20, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "quantile(0.5,\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "Disk space usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The max number of data parts of LSM tree across all storage nodes in the last [partition](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).\nIncrease in number of parts (the hard limit is 512) is an evidence of slow merge performance - check the resource utilization.\n* `indexdb` - inverted index\n* `storage/small` - recently added parts of data ingested into storage (hot data)\n* `storage/big` - small parts gradually merged into bigger parts (cold data)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1102 }, "id": 22, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(vm_parts{job=~\"$job_storage\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(vm_last_partition_parts{job=~\"$job_storage\", instance=~\"$instance\"}) by(type)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "LSM parts max by type ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high [churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "percent" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1110 }, "id": 202, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "indexdb", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "datapoints", "range": true, "refId": "B" } ], "title": "Disk space usage % by type ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How many data points are in RAM queue waiting to be written into storage. The number of pending data points should be in the range from 0 to `5*`, since VictoriaMetrics pushes pending data to persistent storage every 5 seconds. The index datapoints value in general is much lower.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "pending index entries" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "decimals", "value": 3 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1110 }, "id": 14, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"storage\"}[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "pending datapoints", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(max_over_time(vm_pending_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb\"}[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "pending index entries", "range": true, "refId": "B" } ], "title": "Pending datapoints ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows network usage by vmstorage services.\n* Writes show traffic sent to clients.\n* Reads show traffic received from clients.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1118 }, "id": 206, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "read from {{name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) by(name) * 8 > 0", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write to {{name}}", "range": true, "refId": "B" } ], "title": "Network usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Value > 0 means vmstorage is in readonly mode.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1118 }, "id": 141, "links": [ { "targetBlank": true, "title": "Readonly mode", "url": "https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode" } ], "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "vm_storage_is_read_only{job=~\"$job_storage\", instance=~\"$instance\"} > 0", "interval": "", "legendFormat": "{{ instance }}", "range": true, "refId": "A" } ], "title": "Readonly mode", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max number of existing [snapshots](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) across vmstorages.\n\nRecently made snapshots do not occupy disk space. But with time, as snapshots become older, they start to occupy more and more disk space.\n\nIt is recommended deleting old snapshots when they are no longer needed to free up disk space.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1126 }, "id": 216, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(vm_snapshots{job=~\"$job_storage\", instance=~\"$instance\"})", "interval": "", "legendFormat": "snapshots", "range": true, "refId": "A" } ], "title": "Number of snapshots", "type": "timeseries" } ], "title": "vmstorage ($instance)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 42, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Request rate accepted by vmselect nodes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8384 }, "id": 92, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_http_requests_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Requests rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max number of concurrent selects across instances.\n* `max` limit can be configured via `search.maxConcurrentRequests` flag\n* `current` shows the current number of goroutines busy with processing requests\n\nWhen `current` hits `max` constantly, it means one or more vmselect nodes are overloaded with number of requests. If you observe that CPU for vmselects is saturated, consider adding more vmselect replicas or increase CPU resources. If CPU and Memory panels show a plenty of free resources - try increasing `-search.maxConcurrentRequests`. Please note, the higher is `-search.maxConcurrentRequests`, the higher could be [peak memory usage](https://docs.victoriametrics.com/victoriametrics/troubleshooting/#out-of-memory-errors).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8384 }, "id": 95, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(max_over_time(vm_concurrent_select_current{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) ", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "current", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "min(vm_concurrent_select_capacity{job=~\"$job_select\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "max", "refId": "B" } ], "title": "Concurrent selects ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_select&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8392 }, "id": 163, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "CPU usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=$job_select&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8392 }, "id": 165, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "quantile(0.5,\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_select\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "Memory (anon) usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max 99th percentile of time series read per query across all vmselects.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8400 }, "id": 178, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(rate(vm_series_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "series", "range": true, "refId": "A" } ], "title": "Series read per query ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per time series.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8400 }, "id": 180, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(rate(vm_rows_read_per_series_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "datapoints", "range": true, "refId": "A" } ], "title": "Datapoints read per series ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) read per query.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8408 }, "id": 179, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(rate(vm_rows_read_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "datapoints", "range": true, "refId": "A" } ], "title": "Datapoints read per query ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max 99th percentile of number of [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) scanner per query.\n\nThis number can exceed number of DatapointsReadPerQuery if `step` query arg passed to [/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) is smaller than the lookbehind window set in square brackets of [rollup function](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, if `increase(some_metric[1h])` is executed with the `step=5m`, then the same [data samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) on a hour time range are scanned `1h/5m=12` times. See [this article](https://valyala.medium.com/how-to-optimize-promql-and-metricsql-queries-85a1b75bf986) for details.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8408 }, "id": 181, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(rate(vm_rows_scanned_per_query_bucket{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "datapoints", "range": true, "refId": "A" } ], "title": "Datapoints scanned per query ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows network usage between vmselects and clients, such as vmalert, Grafana, vmui, etc.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8416 }, "id": 93, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "read from client", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write to client", "range": true, "refId": "B" } ], "title": "Network usage: clients ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows network usage between vmselects and vmstorages.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8416 }, "id": 207, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "read from vmstorage", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write to vmstorage", "range": true, "refId": "B" } ], "title": "Network usage: vmstorage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the [rollup result cache](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#rollup-result-cache) miss ratio for query when cache is enabled. \nRollup cache is typically hit in two scenarios:\n1. Repeated [range queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) with increasing time, start and end arguments;\n2. Repeated [instant queries](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query) containing rollup functions with lookbehind window exceeding `-search.minWindowForInstantRollupOptimization`.\n\nA lower value indicates high cache utilization, suggesting that most queries are repeated from stable clients such as vmalert rules or Grafana dashboards.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8424 }, "id": 226, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rollup_result_cache_miss_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))\n/\nsum(rate(vm_rollup_result_cache_requests_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "miss", "range": true, "refId": "A" } ], "title": "Rollup result cache miss ratio ($instance)", "type": "timeseries" } ], "title": "vmselect ($instance)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, "id": 40, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8385 }, "id": 97, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_http_requests_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Requests rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the max number of ongoing insertions.\n* `max` - equal to number of CPU * 2 by default. May be configured with `maxConcurrentInserts` flag;\n* `current` - current number of goroutines busy with processing requests.\n\n`-maxConcurrentInserts` limits the number of insert requests which may be actively processed at any given point in time. All the other insert requests are queued for up to `-insert.maxQueueDuration` in the hope they will get a chance to be processed. This queue is used mostly for absorbing spikes for incoming insert request rate.\n\nWhen `current` hits `max` constantly, it means vminsert node is overloaded and requires more CPU or higher limits.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } }, { "id": "custom.fillOpacity", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8385 }, "id": 99, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(max_over_time(vm_concurrent_insert_current{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "current", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "min(vm_concurrent_insert_capacity{job=~\"$job_insert\", instance=~\"$instance\"})", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "refId": "B" } ], "title": "Concurrent inserts ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_insert&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8393 }, "id": 185, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "quantile(0.5,\n rate(process_cpu_seconds_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "CPU usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=$job_insert&var-ds=$ds&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8393 }, "id": 187, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "quantile(0.5,\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_insert\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "median", "range": true, "refId": "C" } ], "title": "Memory (anon) usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the saturation level of connection between vminsert and vmstorage components. \n\nIf the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that either vminsert or vmstorage nodes are struggling with the load. Verify CPU/mem saturation of both components and network saturation between them.\nIf vminsert resources are saturated - consider adding more resources or scale vminserts horizontally.\n\nIf vminsert resources and network are fine, check vmstorage metrics for anomalies.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0.9 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8401 }, "id": 139, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(addr)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Storage connection saturation ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows when vmstorage node is unreachable for vminsert.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8401 }, "id": 114, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "vm_rpc_vmstorage_is_reachable{job=~\"$job\", instance=~\"$instance\"} != 1", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} => {{addr}}", "range": true, "refId": "A" } ], "title": "Storage reachability ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows network usage between vminserts and clients, such as vmagent, Prometheus, or any other client pushing metrics to vminsert.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8409 }, "id": 208, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "read from client", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcplistener_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write to client", "range": true, "refId": "B" } ], "title": "Network usage: clients ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows network usage between vminserts and vmstorages.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read.*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8409 }, "id": 209, "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcpdialer_read_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "read from vmstorage", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_tcpdialer_written_bytes_total{job=~\"$job_insert\", instance=~\"$instance\"}[$__rate_interval])) * 8 > 0", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write to vmstorage", "range": true, "refId": "B" } ], "title": "Network usage: vmstorage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8417 }, "id": 88, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(histogram_quantile(0.99, sum(increase(vm_rows_per_insert_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (instance, vmrange)))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" } ], "title": "Rows per insert ($instance)", "type": "timeseries" } ], "title": "vminsert ($instance)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "id": 194, "panels": [ { "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 44 }, "id": 198, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance, or a single instance in cases like memory breakdown.", "mode": "markdown" }, "pluginVersion": "12.3.0", "title": "", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "id": 189, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "RSS memory usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "id": 190, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "RSS anonymous memory usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "This panel breaks down memory usage by type. It’s intended for inspecting a single instance.\n\nHow to use:\n- Start from the high-level RSS panel.\n- Identify an instance with unexpected or abnormal memory growth.\n- Filter to that instance to inspect the detailed breakdown.\n\nInterpretation\n- A steadily rising Go Heap usually indicates a memory leak. Collect pprof memory profile.\n- A growing Go Stack commonly points to a goroutine leak.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, "id": 225, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "# go other\nmax(\n go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"} +\n go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"} +\n go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"} +\n go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"}\n)", "interval": "", "legendFormat": "Go: Other", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "# go stack\nmax(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Go: Stack", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "#go heap inuse\nmax(\n go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"}\n +\n (\n go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"}\n -\n go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"}\n )\n)", "hide": false, "instant": false, "legendFormat": "Go: Heap", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "# go heap reserved\nmax(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Go: Heap Reserved", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "# vm cache\nmax(sum(vm_cache_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by (job, instance))", "hide": false, "instant": false, "legendFormat": "Mmap: VM Cache", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "# file pages\nmax(process_resident_memory_file_bytes{job=~\"$job\", instance=~\"$instance\"})", "hide": false, "instant": false, "legendFormat": "File cache", "range": true, "refId": "F" } ], "title": "Memory usage breakdown", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the approx time needed to reach 100% of allowed disk capacity based on the following params:\n* free disk space (after -storage.minFreeDiskSpaceBytes);\n* row ingestion rate;\n* compression.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, "id": 196, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"}-vm_free_disk_space_limit_bytes{job=~\"$job_storage\", instance=~\"$instance\"}) \n/ \nignoring(path) (\n (rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) - \n sum(rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])) without(type)) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"}) without(type)\n )\n +\n rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d]) * \n (\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) without(type) /\n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type=\"indexdb/file\"}) without(type)\n )\n) > 0", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Storage full ETA ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 }, "id": 192, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "CPU usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 62 }, "id": 201, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}:{{instance}} (indexdb)", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{job}}:{{instance}} (datapoints)", "range": true, "refId": "B" } ], "title": "Disk space usage by type ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 }, "id": 200, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "", "range": true, "refId": "A" } ], "title": "Disk space usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${ds}" }, "description": "Major page faults occur when required data is not in physical memory and must be loaded from disk (storage or swap). It usually happens with:\n- insufficient physical memory\n- active use of swap space\n- file mapping access\n\nIt may cause significant performance slowdown if sustained high.\n\nPossible workaround:\n- Extend RAM to scale up\n- Reduce swap usage by adjusting vm.swappiness", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 70 }, "id": 224, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${ds}" }, "editorMode": "code", "expr": "sum(rate(process_major_pagefaults_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job,instance) > 0", "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "Major page faults rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "bars", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Source Code", "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 78 }, "id": 203, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.3.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version,location) > 0", "format": "time_series", "hide": false, "interval": "5m", "intervalFactor": 1, "legendFormat": "{{instance}} ({{job}}) - {{level}}: {{location}}", "range": true, "refId": "A" } ], "title": "Logging rate", "type": "timeseries" } ], "title": "Drilldown", "type": "row" } ], "preload": false, "refresh": "", "schemaVersion": 41, "tags": [ "victoriametrics" ], "templating": { "list": [ { "current": { "text": "vmetrics-local", "value": "bfdho2mhaw54wc" }, "includeAll": false, "name": "ds", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", "includeAll": true, "multi": true, "name": "job", "options": [], "query": { "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", "refId": "VictoriaMetrics-job-Variable-Query" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", "hide": 2, "includeAll": true, "multi": true, "name": "job_insert", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", "hide": 2, "includeAll": true, "multi": true, "name": "job_select", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", "hide": 2, "includeAll": true, "multi": true, "name": "job_storage", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", "includeAll": true, "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", "refId": "VictoriaMetrics-instance-Variable-Query" }, "refresh": 1, "regex": "", "type": "query" }, { "baseFilters": [], "datasource": { "type": "prometheus", "uid": "$ds" }, "filters": [], "name": "adhoc", "type": "adhoc" } ] }, "time": { "from": "now-30m", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "VictoriaMetrics - cluster", "uid": "oSselfmonclusterdev01", "version": 3 }