add prometheus instrumentation
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
12
README.md
12
README.md
@@ -33,6 +33,18 @@ Run `swag init --exclude "pkg.mod,pkg.build,pkg.tools" -o server/router/docs`
|
|||||||
- Daily summaries aggregate the hourly snapshots for the day; monthly summaries aggregate daily summaries for the month.
|
- Daily summaries aggregate the hourly snapshots for the day; monthly summaries aggregate daily summaries for the month.
|
||||||
- Snapshots are registered in `snapshot_registry` so regeneration via `/api/snapshots/aggregate` can locate the correct tables (fallback scanning is also supported).
|
- Snapshots are registered in `snapshot_registry` so regeneration via `/api/snapshots/aggregate` can locate the correct tables (fallback scanning is also supported).
|
||||||
- Reports (XLSX with totals/charts) are generated automatically after hourly, daily, and monthly jobs and written to a reports directory.
|
- Reports (XLSX with totals/charts) are generated automatically after hourly, daily, and monthly jobs and written to a reports directory.
|
||||||
|
- Prometheus metrics are exposed at `/metrics`:
|
||||||
|
- Snapshots/aggregations: `vctp_hourly_snapshots_total`, `vctp_hourly_snapshots_failed_total`, `vctp_hourly_snapshot_last_unix`, `vctp_hourly_snapshot_last_rows`, `vctp_daily_aggregations_total`, `vctp_daily_aggregations_failed_total`, `vctp_daily_aggregation_duration_seconds`, `vctp_monthly_aggregations_total`, `vctp_monthly_aggregations_failed_total`, `vctp_monthly_aggregation_duration_seconds`, `vctp_reports_available`
|
||||||
|
- vCenter health/perf: `vctp_vcenter_connect_failures_total{vcenter}`, `vctp_vcenter_snapshot_duration_seconds{vcenter}`, `vctp_vcenter_inventory_size{vcenter}`
|
||||||
|
|
||||||
|
#### RPM Layout (summary)
|
||||||
|
The RPM installs the service and defaults under `/usr/bin`, config under `/etc/dtms`, and data under `/var/lib/vctp`:
|
||||||
|
- Binary: `/usr/bin/vctp-linux-amd64`
|
||||||
|
- Systemd unit: `/etc/systemd/system/vctp.service`
|
||||||
|
- Defaults/env: `/etc/dtms/vctp.yml` (override with `-settings`), `/etc/default/vctp` (environment)
|
||||||
|
- TLS cert/key: `/etc/dtms/vctp.crt` and `/etc/dtms/vctp.key` (generated if absent)
|
||||||
|
- Data: SQLite DB and reports default to `/var/lib/vctp` (reports under `/var/lib/vctp/reports`)
|
||||||
|
- Scripts: preinstall/postinstall handle directory creation and permissions.
|
||||||
|
|
||||||
#### Settings File
|
#### Settings File
|
||||||
Configuration now lives in the YAML settings file. By default the service reads
|
Configuration now lives in the YAML settings file. By default the service reads
|
||||||
|
|||||||
7
go.mod
7
go.mod
@@ -8,6 +8,7 @@ require (
|
|||||||
github.com/jackc/pgx/v5 v5.8.0
|
github.com/jackc/pgx/v5 v5.8.0
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
github.com/jmoiron/sqlx v1.4.0
|
||||||
github.com/pressly/goose/v3 v3.26.0
|
github.com/pressly/goose/v3 v3.26.0
|
||||||
|
github.com/prometheus/client_golang v1.19.0
|
||||||
github.com/swaggo/swag v1.16.6
|
github.com/swaggo/swag v1.16.6
|
||||||
github.com/vmware/govmomi v0.52.0
|
github.com/vmware/govmomi v0.52.0
|
||||||
github.com/xuri/excelize/v2 v2.10.0
|
github.com/xuri/excelize/v2 v2.10.0
|
||||||
@@ -19,6 +20,8 @@ require (
|
|||||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||||
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||||
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
github.com/go-openapi/jsonreference v0.19.6 // indirect
|
||||||
@@ -34,6 +37,9 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
|
github.com/prometheus/client_model v0.5.0 // indirect
|
||||||
|
github.com/prometheus/common v0.48.0 // indirect
|
||||||
|
github.com/prometheus/procfs v0.15.1 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/richardlehane/mscfb v1.0.6 // indirect
|
github.com/richardlehane/mscfb v1.0.6 // indirect
|
||||||
github.com/richardlehane/msoleps v1.0.6 // indirect
|
github.com/richardlehane/msoleps v1.0.6 // indirect
|
||||||
@@ -51,6 +57,7 @@ require (
|
|||||||
golang.org/x/sys v0.40.0 // indirect
|
golang.org/x/sys v0.40.0 // indirect
|
||||||
golang.org/x/text v0.33.0 // indirect
|
golang.org/x/text v0.33.0 // indirect
|
||||||
golang.org/x/tools v0.41.0 // indirect
|
golang.org/x/tools v0.41.0 // indirect
|
||||||
|
google.golang.org/protobuf v1.33.0 // indirect
|
||||||
modernc.org/libc v1.67.4 // indirect
|
modernc.org/libc v1.67.4 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.11.0 // indirect
|
modernc.org/memory v1.11.0 // indirect
|
||||||
|
|||||||
18
go.sum
18
go.sum
@@ -8,6 +8,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
|
|||||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||||
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
|
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
|
||||||
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||||
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
|
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||||
|
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
@@ -77,12 +81,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
|||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
|
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
|
||||||
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
|
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
|
||||||
|
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
|
||||||
|
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
|
||||||
|
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
|
||||||
|
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
|
||||||
|
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
|
||||||
|
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
|
||||||
|
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||||
|
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
|
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
|
||||||
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
|
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
|
||||||
github.com/richardlehane/msoleps v1.0.5 h1:kNlmACZuwC8ZWPLoJtD+HtZOsKJgYn7gXgUIcRB7dbo=
|
|
||||||
github.com/richardlehane/msoleps v1.0.5/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
|
||||||
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
|
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
|
||||||
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
|
||||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||||
@@ -139,6 +149,8 @@ golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
|||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||||
|
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||||
|
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
@@ -173,8 +185,6 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
|||||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||||
modernc.org/sqlite v1.43.0 h1:8YqiFx3G1VhHTXO2Q00bl1Wz9KhS9Q5okwfp9Y97VnA=
|
|
||||||
modernc.org/sqlite v1.43.0/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
|
|
||||||
modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc=
|
modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc=
|
||||||
modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE=
|
modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE=
|
||||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||||
|
|||||||
125
internal/metrics/metrics.go
Normal file
125
internal/metrics/metrics.go
Normal file
@@ -0,0 +1,125 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
registry = prometheus.NewRegistry()
|
||||||
|
|
||||||
|
HourlySnapshotTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_hourly_snapshots_total", Help: "Total number of hourly snapshot jobs completed."})
|
||||||
|
HourlySnapshotFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_hourly_snapshots_failed_total", Help: "Hourly snapshot jobs that failed."})
|
||||||
|
HourlySnapshotLast = prometheus.NewGauge(prometheus.GaugeOpts{Name: "vctp_hourly_snapshot_last_unix", Help: "Unix timestamp of the last hourly snapshot start time."})
|
||||||
|
HourlySnapshotRows = prometheus.NewGauge(prometheus.GaugeOpts{Name: "vctp_hourly_snapshot_last_rows", Help: "Row count of the last hourly snapshot table."})
|
||||||
|
|
||||||
|
DailyAggregationsTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_daily_aggregations_total", Help: "Total number of daily aggregation jobs completed."})
|
||||||
|
DailyAggregationFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_daily_aggregations_failed_total", Help: "Daily aggregation jobs that failed."})
|
||||||
|
DailyAggregationDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||||
|
Name: "vctp_daily_aggregation_duration_seconds",
|
||||||
|
Help: "Duration of daily aggregation jobs.",
|
||||||
|
Buckets: prometheus.ExponentialBuckets(1, 2, 10),
|
||||||
|
})
|
||||||
|
|
||||||
|
MonthlyAggregationsTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_monthly_aggregations_total", Help: "Total number of monthly aggregation jobs completed."})
|
||||||
|
MonthlyAggregationFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_monthly_aggregations_failed_total", Help: "Monthly aggregation jobs that failed."})
|
||||||
|
MonthlyAggregationDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||||
|
Name: "vctp_monthly_aggregation_duration_seconds",
|
||||||
|
Help: "Duration of monthly aggregation jobs.",
|
||||||
|
Buckets: prometheus.ExponentialBuckets(1, 2, 10),
|
||||||
|
})
|
||||||
|
|
||||||
|
ReportsAvailable = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||||
|
Name: "vctp_reports_available",
|
||||||
|
Help: "Number of downloadable reports present on disk.",
|
||||||
|
})
|
||||||
|
|
||||||
|
VcenterConnectFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||||
|
Name: "vctp_vcenter_connect_failures_total",
|
||||||
|
Help: "Failed connections to vCenter during snapshot runs.",
|
||||||
|
}, []string{"vcenter"})
|
||||||
|
|
||||||
|
VcenterSnapshotDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
|
Name: "vctp_vcenter_snapshot_duration_seconds",
|
||||||
|
Help: "Duration of per-vCenter hourly snapshot jobs.",
|
||||||
|
Buckets: prometheus.ExponentialBuckets(0.5, 2, 10),
|
||||||
|
}, []string{"vcenter"})
|
||||||
|
|
||||||
|
VcenterInventorySize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
|
Name: "vctp_vcenter_inventory_size",
|
||||||
|
Help: "Number of VMs seen in the last successful snapshot per vCenter.",
|
||||||
|
}, []string{"vcenter"})
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
registry.MustRegister(
|
||||||
|
HourlySnapshotTotal,
|
||||||
|
HourlySnapshotFailures,
|
||||||
|
HourlySnapshotLast,
|
||||||
|
HourlySnapshotRows,
|
||||||
|
DailyAggregationsTotal,
|
||||||
|
DailyAggregationFailures,
|
||||||
|
DailyAggregationDuration,
|
||||||
|
MonthlyAggregationsTotal,
|
||||||
|
MonthlyAggregationFailures,
|
||||||
|
MonthlyAggregationDuration,
|
||||||
|
ReportsAvailable,
|
||||||
|
VcenterConnectFailures,
|
||||||
|
VcenterSnapshotDuration,
|
||||||
|
VcenterInventorySize,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler returns an http.Handler that serves Prometheus metrics.
|
||||||
|
func Handler() http.Handler {
|
||||||
|
return promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordVcenterSnapshot logs per-vCenter snapshot metrics.
|
||||||
|
func RecordVcenterSnapshot(vcenter string, duration time.Duration, vmCount int64, err error) {
|
||||||
|
VcenterSnapshotDuration.WithLabelValues(vcenter).Observe(duration.Seconds())
|
||||||
|
if err != nil {
|
||||||
|
VcenterConnectFailures.WithLabelValues(vcenter).Inc()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
VcenterInventorySize.WithLabelValues(vcenter).Set(float64(vmCount))
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHourlySnapshot logs aggregate hourly snapshot results.
|
||||||
|
func RecordHourlySnapshot(start time.Time, rows int64, err error) {
|
||||||
|
HourlySnapshotLast.Set(float64(start.Unix()))
|
||||||
|
HourlySnapshotRows.Set(float64(rows))
|
||||||
|
if err != nil {
|
||||||
|
HourlySnapshotFailures.Inc()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
HourlySnapshotTotal.Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordDailyAggregation logs daily aggregation metrics.
|
||||||
|
func RecordDailyAggregation(duration time.Duration, err error) {
|
||||||
|
DailyAggregationDuration.Observe(duration.Seconds())
|
||||||
|
if err != nil {
|
||||||
|
DailyAggregationFailures.Inc()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
DailyAggregationsTotal.Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordMonthlyAggregation logs monthly aggregation metrics.
|
||||||
|
func RecordMonthlyAggregation(duration time.Duration, err error) {
|
||||||
|
MonthlyAggregationDuration.Observe(duration.Seconds())
|
||||||
|
if err != nil {
|
||||||
|
MonthlyAggregationFailures.Inc()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
MonthlyAggregationsTotal.Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetReportsAvailable updates the gauge for report files found on disk.
|
||||||
|
func SetReportsAvailable(count int) {
|
||||||
|
ReportsAvailable.Set(float64(count))
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
"vctp/db"
|
"vctp/db"
|
||||||
|
"vctp/internal/metrics"
|
||||||
"vctp/internal/report"
|
"vctp/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -27,6 +28,7 @@ func (c *CronTask) AggregateDailySummary(ctx context.Context, date time.Time, fo
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Time, force bool) error {
|
func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Time, force bool) error {
|
||||||
|
jobStart := time.Now()
|
||||||
dayStart := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location())
|
dayStart := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location())
|
||||||
dayEnd := dayStart.AddDate(0, 0, 1)
|
dayEnd := dayStart.AddDate(0, 0, 1)
|
||||||
summaryTable, err := dailySummaryTableName(targetTime)
|
summaryTable, err := dailySummaryTableName(targetTime)
|
||||||
@@ -133,9 +135,12 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
|||||||
|
|
||||||
if err := c.generateReport(ctx, summaryTable); err != nil {
|
if err := c.generateReport(ctx, summaryTable); err != nil {
|
||||||
c.Logger.Warn("failed to generate daily report", "error", err, "table", summaryTable)
|
c.Logger.Warn("failed to generate daily report", "error", err, "table", summaryTable)
|
||||||
|
metrics.RecordDailyAggregation(time.Since(jobStart), err)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable)
|
c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable)
|
||||||
|
metrics.RecordDailyAggregation(time.Since(jobStart), nil)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
"vctp/db"
|
"vctp/db"
|
||||||
"vctp/db/queries"
|
"vctp/db/queries"
|
||||||
|
"vctp/internal/metrics"
|
||||||
"vctp/internal/report"
|
"vctp/internal/report"
|
||||||
"vctp/internal/utils"
|
"vctp/internal/utils"
|
||||||
"vctp/internal/vcenter"
|
"vctp/internal/vcenter"
|
||||||
@@ -168,6 +169,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
|||||||
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
metrics.RecordHourlySnapshot(startTime, rowCount, err)
|
||||||
if err := c.generateReport(ctx, tableName); err != nil {
|
if err := c.generateReport(ctx, tableName); err != nil {
|
||||||
c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName)
|
c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName)
|
||||||
}
|
}
|
||||||
@@ -636,44 +638,6 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName string, row inventorySnapshotRow) error {
|
|
||||||
query := fmt.Sprintf(`
|
|
||||||
INSERT INTO %s (
|
|
||||||
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
|
|
||||||
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
|
|
||||||
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
|
|
||||||
)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
|
||||||
`, tableName)
|
|
||||||
|
|
||||||
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
|
|
||||||
|
|
||||||
_, err := dbConn.ExecContext(ctx, query,
|
|
||||||
row.InventoryId,
|
|
||||||
row.Name,
|
|
||||||
row.Vcenter,
|
|
||||||
row.VmId,
|
|
||||||
row.EventKey,
|
|
||||||
row.CloudId,
|
|
||||||
row.CreationTime,
|
|
||||||
row.DeletionTime,
|
|
||||||
row.ResourcePool,
|
|
||||||
row.Datacenter,
|
|
||||||
row.Cluster,
|
|
||||||
row.Folder,
|
|
||||||
row.ProvisionedDisk,
|
|
||||||
row.VcpuCount,
|
|
||||||
row.RamGB,
|
|
||||||
row.IsTemplate,
|
|
||||||
row.PoweredOn,
|
|
||||||
row.SrmPlaceholder,
|
|
||||||
row.VmUuid,
|
|
||||||
row.SnapshotTime,
|
|
||||||
row.IsPresent,
|
|
||||||
)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func insertHourlyBatch(ctx context.Context, dbConn *sqlx.DB, tableName string, rows []inventorySnapshotRow) error {
|
func insertHourlyBatch(ctx context.Context, dbConn *sqlx.DB, tableName string, rows []inventorySnapshotRow) error {
|
||||||
if len(rows) == 0 {
|
if len(rows) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -727,9 +691,11 @@ INSERT INTO %s (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string) error {
|
func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string) error {
|
||||||
|
started := time.Now()
|
||||||
c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url)
|
c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url)
|
||||||
vc := vcenter.New(c.Logger, c.VcCreds)
|
vc := vcenter.New(c.Logger, c.VcCreds)
|
||||||
if err := vc.Login(url); err != nil {
|
if err := vc.Login(url); err != nil {
|
||||||
|
metrics.RecordVcenterSnapshot(url, time.Since(started), 0, err)
|
||||||
return fmt.Errorf("unable to connect to vcenter: %w", err)
|
return fmt.Errorf("unable to connect to vcenter: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
@@ -740,6 +706,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
|||||||
|
|
||||||
vcVms, err := vc.GetAllVMsWithProps()
|
vcVms, err := vc.GetAllVMsWithProps()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
metrics.RecordVcenterSnapshot(url, time.Since(started), 0, err)
|
||||||
return fmt.Errorf("unable to get VMs from vcenter: %w", err)
|
return fmt.Errorf("unable to get VMs from vcenter: %w", err)
|
||||||
}
|
}
|
||||||
canDetectMissing := len(vcVms) > 0
|
canDetectMissing := len(vcVms) > 0
|
||||||
@@ -856,6 +823,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := insertHourlyBatch(ctx, dbConn, tableName, batch); err != nil {
|
if err := insertHourlyBatch(ctx, dbConn, tableName, batch); err != nil {
|
||||||
|
metrics.RecordVcenterSnapshot(url, time.Since(started), totals.VmCount, err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -866,6 +834,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
|||||||
"ram_total_gb", totals.RamTotal,
|
"ram_total_gb", totals.RamTotal,
|
||||||
"disk_total_gb", totals.DiskTotal,
|
"disk_total_gb", totals.DiskTotal,
|
||||||
)
|
)
|
||||||
|
metrics.RecordVcenterSnapshot(url, time.Since(started), totals.VmCount, nil)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
"vctp/db"
|
"vctp/db"
|
||||||
|
"vctp/internal/metrics"
|
||||||
"vctp/internal/report"
|
"vctp/internal/report"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,6 +30,7 @@ func (c *CronTask) AggregateMonthlySummary(ctx context.Context, month time.Time,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time.Time, force bool) error {
|
func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time.Time, force bool) error {
|
||||||
|
jobStart := time.Now()
|
||||||
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
|
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -107,9 +109,12 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
|
|||||||
|
|
||||||
if err := c.generateReport(ctx, monthlyTable); err != nil {
|
if err := c.generateReport(ctx, monthlyTable); err != nil {
|
||||||
c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable)
|
c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable)
|
||||||
|
metrics.RecordMonthlyAggregation(time.Since(jobStart), err)
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable)
|
c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable)
|
||||||
|
metrics.RecordMonthlyAggregation(time.Since(jobStart), nil)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
17
server/handler/metrics.go
Normal file
17
server/handler/metrics.go
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"vctp/internal/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Metrics exposes Prometheus metrics.
|
||||||
|
// @Summary Prometheus metrics
|
||||||
|
// @Description Exposes Prometheus metrics for vctp.
|
||||||
|
// @Tags metrics
|
||||||
|
// @Produce plain
|
||||||
|
// @Success 200 "Prometheus metrics"
|
||||||
|
// @Router /metrics [get]
|
||||||
|
func (h *Handler) Metrics(w http.ResponseWriter, r *http.Request) {
|
||||||
|
metrics.Handler().ServeHTTP(w, r)
|
||||||
|
}
|
||||||
@@ -65,6 +65,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
|
|||||||
mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce)
|
mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce)
|
||||||
mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate)
|
mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate)
|
||||||
mux.HandleFunc("/api/snapshots/regenerate-hourly-reports", h.SnapshotRegenerateHourlyReports)
|
mux.HandleFunc("/api/snapshots/regenerate-hourly-reports", h.SnapshotRegenerateHourlyReports)
|
||||||
|
mux.HandleFunc("/metrics", h.Metrics)
|
||||||
|
|
||||||
mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList)
|
mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList)
|
||||||
mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList)
|
mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList)
|
||||||
|
|||||||
Reference in New Issue
Block a user