diff --git a/systems/makanek/monitoring/default.nix b/systems/makanek/monitoring/default.nix index 916ed8a..fbb025b 100644 --- a/systems/makanek/monitoring/default.nix +++ b/systems/makanek/monitoring/default.nix @@ -10,10 +10,39 @@ in { services.grafana = { enable = true; - settings.server = { - domain = "grafana.kmein.r"; - http_port = 9444; - http_addr = "127.0.0.1"; + settings = { + server = { + domain = "grafana.kmein.r"; + http_port = 9444; + http_addr = "127.0.0.1"; + }; + smtp = let + inherit (import {inherit lib;}) cock; + address = builtins.split "@" cock.user; + in { + enabled = true; + from_address = cock.address; + password = cock.password; + user = cock.user; + host = cock.smtpSettings cock.smtp; + startTLS_policy = "MandatoryStartTLS"; + }; + dashboards.default_home_dashboard_path = toString ./grafana-dashboards/niveum.json; + security = { + admin_user = "admin"; + admin_password = lib.strings.fileContents ; + }; + }; + provision = { + enable = true; + dashboards.settings.providers = [ + { + name = "dashboards"; + type = "file"; + options.path = ./grafana-dashboards; + } + ]; + datasources.settings.datasources = builtins.fromJSON (builtins.readFile ./grafana-datasources.json); }; }; diff --git a/systems/makanek/monitoring/grafana-dashboards/niveum.json b/systems/makanek/monitoring/grafana-dashboards/niveum.json new file mode 100644 index 0000000..a45ad3b --- /dev/null +++ b/systems/makanek/monitoring/grafana-dashboards/niveum.json @@ -0,0 +1,944 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "refId": "A" + } + ], + "title": "Running", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "sAagoeS7k" + }, + "fieldConfig": { + "defaults": { + "unit": "cps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "sAagoeS7k" + }, + "expr": "rate({job=\"systemd-journal\"} |~ \"[Ee]rror|[Cc]rit\" [5m])", + "legendFormat": "{{unit}}@{{host}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "cps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "sum(node_systemd_unit_state{state!=\"inactive\",state!=\"activating\",state!=\"deactivating\"}) by (state)", + "interval": "", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "SystemD Units", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "sum(node_logind_sessions{type=\"tty\"}) by (job)", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "tty sessions", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "loki", + "uid": "sAagoeS7k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "sAagoeS7k" + }, + "expr": "rate({unit=\"nginx.service\"} |= \"GET\"[5m])", + "legendFormat": "{{host}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "nginx requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "refId": "A" + } + ], + "title": "Free", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "node_filesystem_avail_bytes{mountpoint=~\"/|/backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{job}}:{{mountpoint}}", + "refId": "A" + } + ], + "title": "Free Disk Space", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "node_memory_MemFree_bytes", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Free Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "node_load15", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Load (15m)", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 16, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "refId": "A" + } + ], + "title": "IO", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])", + "interval": "", + "legendFormat": "recv: {{job}} / {{device}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])", + "hide": false, + "interval": "", + "legendFormat": "send: {{job}} / {{device}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network IO", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:2175", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:2176", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "rate(node_disk_written_bytes_total{device=\"sda\"}[5m])", + "interval": "", + "legendFormat": "write: {{job}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "rate(node_disk_read_bytes_total{device=\"sda\"}[5m])", + "hide": false, + "interval": "", + "legendFormat": "read: {{job}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IO", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1290", + "format": "Bps", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1291", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "refId": "A" + } + ], + "title": "HTTP", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 200 + }, + { + "color": "red", + "value": 400 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 34, + "maxDataPoints": 200, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "tOBnpeI7z" + }, + "exemplar": true, + "expr": "probe_http_status_code", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "HTTP Probe", + "type": "status-history" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "niveum", + "uid": "alpUteInz", + "version": 31, + "weekStart": "" +} diff --git a/systems/makanek/monitoring/grafana-datasources.json b/systems/makanek/monitoring/grafana-datasources.json new file mode 100644 index 0000000..8bb9f37 --- /dev/null +++ b/systems/makanek/monitoring/grafana-datasources.json @@ -0,0 +1,38 @@ +[ + { + "id": 2, + "uid": "sAagoeS7k", + "orgId": 1, + "name": "Loki", + "type": "loki", + "typeName": "Loki", + "typeLogoUrl": "public/app/plugins/datasource/loki/img/loki_icon.svg", + "access": "proxy", + "url": "http://localhost:3100", + "user": "", + "database": "", + "basicAuth": false, + "isDefault": false, + "jsonData": {}, + "readOnly": true + }, + { + "id": 1, + "uid": "tOBnpeI7z", + "orgId": 1, + "name": "Prometheus", + "type": "prometheus", + "typeName": "Prometheus", + "typeLogoUrl": "public/app/plugins/datasource/prometheus/img/prometheus_logo.svg", + "access": "proxy", + "url": "http://localhost:9001", + "user": "", + "database": "", + "basicAuth": false, + "isDefault": true, + "jsonData": { + "httpMethod": "POST" + }, + "readOnly": true + } +]