diff --git a/CHANGELOG.md b/CHANGELOG.md index dc49aca..afd1830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Grafana dashboard for adapter metrics (`charts/dashboards/hyperfleet-adapter.json`) — covers events processed, processing duration, errors by type, resource deletions, and adapter health ([HYPERFLEET-1360](https://issues.redhat.com/browse/HYPERFLEET-1360)) + ## [0.2.0] - 2026-03-30 ### Added diff --git a/charts/Chart.yaml b/charts/Chart.yaml index b931ea7..65e3eeb 100644 --- a/charts/Chart.yaml +++ b/charts/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: hyperfleet-adapter description: HyperFleet Adapter - Event-driven adapter services for HyperFleet cluster provisioning type: application -version: 2.0.0 +version: 2.1.0 appVersion: "0.0.0-dev" maintainers: - name: HyperFleet Team diff --git a/charts/README.md b/charts/README.md index 195aaf1..21f6e09 100644 --- a/charts/README.md +++ b/charts/README.md @@ -1,6 +1,6 @@ # hyperfleet-adapter -![Version: 2.0.0](https://img.shields.io/badge/Version-2.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) +![Version: 2.1.0](https://img.shields.io/badge/Version-2.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.0-dev](https://img.shields.io/badge/AppVersion-0.0.0--dev-informational?style=flat-square) HyperFleet Adapter - Event-driven adapter services for HyperFleet cluster provisioning diff --git a/charts/dashboards/hyperfleet-adapter.json b/charts/dashboards/hyperfleet-adapter.json new file mode 100644 index 0000000..ffac285 --- /dev/null +++ b/charts/dashboards/hyperfleet-adapter.json @@ -0,0 +1,908 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(hyperfleet_adapter_events_processed_total{adapter_name=~\"$adapter_name\"}[$__rate_interval])", + "legendFormat": "{{adapter_name}} - {{status}}", + "refId": "A" + } + ], + "title": "Events Processed Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Down" + }, + "1": { + "color": "green", + "index": 1, + "text": "Up" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "1": { + "color": "blue", + "index": 0, + "text": "Active" + } + } + } + ] + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "hyperfleet_adapter_up", + "legendFormat": "Adapter Status", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "hyperfleet_adapter_build_info", + "legendFormat": "Build: {{version}}", + "refId": "B" + } + ], + "title": "Adapter Status", + "description": "Reflects all scraped adapter instances — not filtered by the adapter_name variable, since hyperfleet_adapter_up/build_info carry no adapter_name label.", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum(rate(hyperfleet_adapter_event_processing_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name))", + "legendFormat": "p50 - {{adapter_name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(hyperfleet_adapter_event_processing_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name))", + "legendFormat": "p95 - {{adapter_name}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(hyperfleet_adapter_event_processing_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name))", + "legendFormat": "p99 - {{adapter_name}}", + "refId": "C" + } + ], + "title": "Event Processing Duration (Percentiles)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(hyperfleet_adapter_errors_total{adapter_name=~\"$adapter_name\"}[$__rate_interval])", + "legendFormat": "{{adapter_name}} - {{error_type}}", + "refId": "A" + } + ], + "title": "Error Rate by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "last", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(hyperfleet_adapter_resources_deleted_total{adapter_name=~\"$adapter_name\"}[$__rate_interval])", + "legendFormat": "{{adapter_name}} - {{resource_type}} - {{status}}", + "refId": "A" + } + ], + "title": "Resource Deletion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "last", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "hyperfleet_adapter_resource_deletions_in_progress{adapter_name=~\"$adapter_name\"}", + "legendFormat": "{{adapter_name}} - {{resource_type}}", + "refId": "A" + } + ], + "title": "Deletions In Progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.50, sum(rate(hyperfleet_adapter_resource_deletion_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name, resource_type))", + "legendFormat": "p50 - {{adapter_name}} - {{resource_type}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.95, sum(rate(hyperfleet_adapter_resource_deletion_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name, resource_type))", + "legendFormat": "p95 - {{adapter_name}} - {{resource_type}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum(rate(hyperfleet_adapter_resource_deletion_duration_seconds_bucket{adapter_name=~\"$adapter_name\"}[$__rate_interval])) by (le, adapter_name, resource_type))", + "legendFormat": "p99 - {{adapter_name}} - {{resource_type}}", + "refId": "C" + } + ], + "title": "Deletion Duration (Percentiles)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (resource_type) (rate(hyperfleet_adapter_resources_deleted_total{adapter_name=~\"$adapter_name\"}[$__rate_interval]))", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "title": "Deletion Rate by Resource Type", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Deletions/sec", + "resource_type": "Resource Type" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "hyperfleet", + "adapter", + "observability" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(hyperfleet_adapter_events_processed_total, adapter_name)", + "hide": 0, + "includeAll": true, + "label": "Adapter Name", + "multi": true, + "name": "adapter_name", + "options": [], + "query": { + "query": "label_values(hyperfleet_adapter_events_processed_total, adapter_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HyperFleet Adapter Metrics", + "uid": "hyperfleet-adapter", + "version": 1, + "weekStart": "" +} diff --git a/docs/metrics.md b/docs/metrics.md index 39d5a22..429ca44 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -10,14 +10,14 @@ The Helm chart includes a **ServiceMonitor** template for automatic discovery by The adapter exposes Prometheus metrics following the [HyperFleet Metrics Standard](https://github.com/openshift-hyperfleet/architecture/blob/main/hyperfleet/standards/metrics.md) with the `hyperfleet_adapter_` prefix. -All adapter metrics include `component`, `version`, and `adapter_name` as constant labels. +All adapter metrics include `component` and `version` as constant labels. Event-processing and resource-deletion metrics also include `adapter_name`; the baseline metrics (`up`, `build_info`) do not. ### Baseline Metrics | Metric | Type | Labels | Description | |--------|------|--------|-------------| -| `hyperfleet_adapter_build_info` | Gauge | `component`, `version`, `adapter_name`, `commit` | Build information (always 1) | -| `hyperfleet_adapter_up` | Gauge | `component`, `version`, `adapter_name` | Whether the adapter is up and running (1=up, 0=shutting down) | +| `hyperfleet_adapter_build_info` | Gauge | `component`, `version`, `commit` | Build information (always 1) | +| `hyperfleet_adapter_up` | Gauge | `component`, `version` | Whether the adapter is up and running (1=up, 0=shutting down) | ### Event Processing Metrics @@ -120,3 +120,18 @@ These metrics use the `hyperfleet_broker_` prefix and include the adapter's `com ## Alerting and Monitoring For recommended alerting rules, thresholds, and operational PromQL queries, see [alerts.md](alerts.md). + +## Dashboard + +A pre-built Grafana dashboard is shipped at `charts/dashboards/hyperfleet-adapter.json` (repo root). It covers all `hyperfleet_adapter_*` metrics across 8 panels: events processed rate, adapter status, event processing duration percentiles, error rate by type, resource deletion rate, deletions in progress, deletion duration percentiles, and deletion rate by resource type. + +### Importing + +1. Navigate to **Grafana > Dashboards > Import** +2. Upload `charts/dashboards/hyperfleet-adapter.json` +3. Select your Prometheus datasource from the dropdown +4. Click **Import** + +The dashboard uses a parameterized `datasource` variable (no hardcoded UIDs) and an `adapter_name` multi-select variable to filter most panels by adapter instance. The "Adapter Status" panel always reflects all instances, since the underlying `hyperfleet_adapter_up`/`build_info` metrics carry no `adapter_name` label. + +Automated provisioning via `hyperfleet-infra` is tracked in [HYPERFLEET-1363](https://issues.redhat.com/browse/HYPERFLEET-1363).