add prometheus instrumentation
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-15 11:43:29 +11:00
parent 4d754ee263
commit ea68331208
9 changed files with 193 additions and 42 deletions

View File

@@ -33,6 +33,18 @@ Run `swag init --exclude "pkg.mod,pkg.build,pkg.tools" -o server/router/docs`
- Daily summaries aggregate the hourly snapshots for the day; monthly summaries aggregate daily summaries for the month. - Daily summaries aggregate the hourly snapshots for the day; monthly summaries aggregate daily summaries for the month.
- Snapshots are registered in `snapshot_registry` so regeneration via `/api/snapshots/aggregate` can locate the correct tables (fallback scanning is also supported). - Snapshots are registered in `snapshot_registry` so regeneration via `/api/snapshots/aggregate` can locate the correct tables (fallback scanning is also supported).
- Reports (XLSX with totals/charts) are generated automatically after hourly, daily, and monthly jobs and written to a reports directory. - Reports (XLSX with totals/charts) are generated automatically after hourly, daily, and monthly jobs and written to a reports directory.
- Prometheus metrics are exposed at `/metrics`:
- Snapshots/aggregations: `vctp_hourly_snapshots_total`, `vctp_hourly_snapshots_failed_total`, `vctp_hourly_snapshot_last_unix`, `vctp_hourly_snapshot_last_rows`, `vctp_daily_aggregations_total`, `vctp_daily_aggregations_failed_total`, `vctp_daily_aggregation_duration_seconds`, `vctp_monthly_aggregations_total`, `vctp_monthly_aggregations_failed_total`, `vctp_monthly_aggregation_duration_seconds`, `vctp_reports_available`
- vCenter health/perf: `vctp_vcenter_connect_failures_total{vcenter}`, `vctp_vcenter_snapshot_duration_seconds{vcenter}`, `vctp_vcenter_inventory_size{vcenter}`
#### RPM Layout (summary)
The RPM installs the service and defaults under `/usr/bin`, config under `/etc/dtms`, and data under `/var/lib/vctp`:
- Binary: `/usr/bin/vctp-linux-amd64`
- Systemd unit: `/etc/systemd/system/vctp.service`
- Defaults/env: `/etc/dtms/vctp.yml` (override with `-settings`), `/etc/default/vctp` (environment)
- TLS cert/key: `/etc/dtms/vctp.crt` and `/etc/dtms/vctp.key` (generated if absent)
- Data: SQLite DB and reports default to `/var/lib/vctp` (reports under `/var/lib/vctp/reports`)
- Scripts: preinstall/postinstall handle directory creation and permissions.
#### Settings File #### Settings File
Configuration now lives in the YAML settings file. By default the service reads Configuration now lives in the YAML settings file. By default the service reads

7
go.mod
View File

@@ -8,6 +8,7 @@ require (
github.com/jackc/pgx/v5 v5.8.0 github.com/jackc/pgx/v5 v5.8.0
github.com/jmoiron/sqlx v1.4.0 github.com/jmoiron/sqlx v1.4.0
github.com/pressly/goose/v3 v3.26.0 github.com/pressly/goose/v3 v3.26.0
github.com/prometheus/client_golang v1.19.0
github.com/swaggo/swag v1.16.6 github.com/swaggo/swag v1.16.6
github.com/vmware/govmomi v0.52.0 github.com/vmware/govmomi v0.52.0
github.com/xuri/excelize/v2 v2.10.0 github.com/xuri/excelize/v2 v2.10.0
@@ -19,6 +20,8 @@ require (
github.com/KyleBanks/depth v1.2.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect
@@ -34,6 +37,9 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect github.com/mfridman/interpolate v0.0.2 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.6 // indirect github.com/richardlehane/mscfb v1.0.6 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect github.com/richardlehane/msoleps v1.0.6 // indirect
@@ -51,6 +57,7 @@ require (
golang.org/x/sys v0.40.0 // indirect golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect golang.org/x/text v0.33.0 // indirect
golang.org/x/tools v0.41.0 // indirect golang.org/x/tools v0.41.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
modernc.org/libc v1.67.4 // indirect modernc.org/libc v1.67.4 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect modernc.org/memory v1.11.0 // indirect

18
go.sum
View File

@@ -8,6 +8,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg= github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo= github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -77,12 +81,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8= github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8=
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
github.com/richardlehane/msoleps v1.0.5 h1:kNlmACZuwC8ZWPLoJtD+HtZOsKJgYn7gXgUIcRB7dbo=
github.com/richardlehane/msoleps v1.0.5/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -139,6 +149,8 @@ golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -173,8 +185,6 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.43.0 h1:8YqiFx3G1VhHTXO2Q00bl1Wz9KhS9Q5okwfp9Y97VnA=
modernc.org/sqlite v1.43.0/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc= modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc=
modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE= modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=

125
internal/metrics/metrics.go Normal file
View File

@@ -0,0 +1,125 @@
package metrics
import (
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
registry = prometheus.NewRegistry()
HourlySnapshotTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_hourly_snapshots_total", Help: "Total number of hourly snapshot jobs completed."})
HourlySnapshotFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_hourly_snapshots_failed_total", Help: "Hourly snapshot jobs that failed."})
HourlySnapshotLast = prometheus.NewGauge(prometheus.GaugeOpts{Name: "vctp_hourly_snapshot_last_unix", Help: "Unix timestamp of the last hourly snapshot start time."})
HourlySnapshotRows = prometheus.NewGauge(prometheus.GaugeOpts{Name: "vctp_hourly_snapshot_last_rows", Help: "Row count of the last hourly snapshot table."})
DailyAggregationsTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_daily_aggregations_total", Help: "Total number of daily aggregation jobs completed."})
DailyAggregationFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_daily_aggregations_failed_total", Help: "Daily aggregation jobs that failed."})
DailyAggregationDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "vctp_daily_aggregation_duration_seconds",
Help: "Duration of daily aggregation jobs.",
Buckets: prometheus.ExponentialBuckets(1, 2, 10),
})
MonthlyAggregationsTotal = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_monthly_aggregations_total", Help: "Total number of monthly aggregation jobs completed."})
MonthlyAggregationFailures = prometheus.NewCounter(prometheus.CounterOpts{Name: "vctp_monthly_aggregations_failed_total", Help: "Monthly aggregation jobs that failed."})
MonthlyAggregationDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "vctp_monthly_aggregation_duration_seconds",
Help: "Duration of monthly aggregation jobs.",
Buckets: prometheus.ExponentialBuckets(1, 2, 10),
})
ReportsAvailable = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "vctp_reports_available",
Help: "Number of downloadable reports present on disk.",
})
VcenterConnectFailures = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "vctp_vcenter_connect_failures_total",
Help: "Failed connections to vCenter during snapshot runs.",
}, []string{"vcenter"})
VcenterSnapshotDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "vctp_vcenter_snapshot_duration_seconds",
Help: "Duration of per-vCenter hourly snapshot jobs.",
Buckets: prometheus.ExponentialBuckets(0.5, 2, 10),
}, []string{"vcenter"})
VcenterInventorySize = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "vctp_vcenter_inventory_size",
Help: "Number of VMs seen in the last successful snapshot per vCenter.",
}, []string{"vcenter"})
)
func init() {
registry.MustRegister(
HourlySnapshotTotal,
HourlySnapshotFailures,
HourlySnapshotLast,
HourlySnapshotRows,
DailyAggregationsTotal,
DailyAggregationFailures,
DailyAggregationDuration,
MonthlyAggregationsTotal,
MonthlyAggregationFailures,
MonthlyAggregationDuration,
ReportsAvailable,
VcenterConnectFailures,
VcenterSnapshotDuration,
VcenterInventorySize,
)
}
// Handler returns an http.Handler that serves Prometheus metrics.
func Handler() http.Handler {
return promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
}
// RecordVcenterSnapshot logs per-vCenter snapshot metrics.
func RecordVcenterSnapshot(vcenter string, duration time.Duration, vmCount int64, err error) {
VcenterSnapshotDuration.WithLabelValues(vcenter).Observe(duration.Seconds())
if err != nil {
VcenterConnectFailures.WithLabelValues(vcenter).Inc()
return
}
VcenterInventorySize.WithLabelValues(vcenter).Set(float64(vmCount))
}
// RecordHourlySnapshot logs aggregate hourly snapshot results.
func RecordHourlySnapshot(start time.Time, rows int64, err error) {
HourlySnapshotLast.Set(float64(start.Unix()))
HourlySnapshotRows.Set(float64(rows))
if err != nil {
HourlySnapshotFailures.Inc()
return
}
HourlySnapshotTotal.Inc()
}
// RecordDailyAggregation logs daily aggregation metrics.
func RecordDailyAggregation(duration time.Duration, err error) {
DailyAggregationDuration.Observe(duration.Seconds())
if err != nil {
DailyAggregationFailures.Inc()
return
}
DailyAggregationsTotal.Inc()
}
// RecordMonthlyAggregation logs monthly aggregation metrics.
func RecordMonthlyAggregation(duration time.Duration, err error) {
MonthlyAggregationDuration.Observe(duration.Seconds())
if err != nil {
MonthlyAggregationFailures.Inc()
return
}
MonthlyAggregationsTotal.Inc()
}
// SetReportsAvailable updates the gauge for report files found on disk.
func SetReportsAvailable(count int) {
ReportsAvailable.Set(float64(count))
}

View File

@@ -6,6 +6,7 @@ import (
"log/slog" "log/slog"
"time" "time"
"vctp/db" "vctp/db"
"vctp/internal/metrics"
"vctp/internal/report" "vctp/internal/report"
) )
@@ -27,6 +28,7 @@ func (c *CronTask) AggregateDailySummary(ctx context.Context, date time.Time, fo
} }
func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Time, force bool) error { func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Time, force bool) error {
jobStart := time.Now()
dayStart := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location()) dayStart := time.Date(targetTime.Year(), targetTime.Month(), targetTime.Day(), 0, 0, 0, 0, targetTime.Location())
dayEnd := dayStart.AddDate(0, 0, 1) dayEnd := dayStart.AddDate(0, 0, 1)
summaryTable, err := dailySummaryTableName(targetTime) summaryTable, err := dailySummaryTableName(targetTime)
@@ -133,9 +135,12 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
if err := c.generateReport(ctx, summaryTable); err != nil { if err := c.generateReport(ctx, summaryTable); err != nil {
c.Logger.Warn("failed to generate daily report", "error", err, "table", summaryTable) c.Logger.Warn("failed to generate daily report", "error", err, "table", summaryTable)
metrics.RecordDailyAggregation(time.Since(jobStart), err)
return err
} }
c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable) c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable)
metrics.RecordDailyAggregation(time.Since(jobStart), nil)
return nil return nil
} }

View File

@@ -12,6 +12,7 @@ import (
"time" "time"
"vctp/db" "vctp/db"
"vctp/db/queries" "vctp/db/queries"
"vctp/internal/metrics"
"vctp/internal/report" "vctp/internal/report"
"vctp/internal/utils" "vctp/internal/utils"
"vctp/internal/vcenter" "vctp/internal/vcenter"
@@ -168,6 +169,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName) c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
} }
metrics.RecordHourlySnapshot(startTime, rowCount, err)
if err := c.generateReport(ctx, tableName); err != nil { if err := c.generateReport(ctx, tableName); err != nil {
c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName) c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName)
} }
@@ -636,44 +638,6 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
} }
} }
func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName string, row inventorySnapshotRow) error {
query := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`, tableName)
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
_, err := dbConn.ExecContext(ctx, query,
row.InventoryId,
row.Name,
row.Vcenter,
row.VmId,
row.EventKey,
row.CloudId,
row.CreationTime,
row.DeletionTime,
row.ResourcePool,
row.Datacenter,
row.Cluster,
row.Folder,
row.ProvisionedDisk,
row.VcpuCount,
row.RamGB,
row.IsTemplate,
row.PoweredOn,
row.SrmPlaceholder,
row.VmUuid,
row.SnapshotTime,
row.IsPresent,
)
return err
}
func insertHourlyBatch(ctx context.Context, dbConn *sqlx.DB, tableName string, rows []inventorySnapshotRow) error { func insertHourlyBatch(ctx context.Context, dbConn *sqlx.DB, tableName string, rows []inventorySnapshotRow) error {
if len(rows) == 0 { if len(rows) == 0 {
return nil return nil
@@ -727,9 +691,11 @@ INSERT INTO %s (
} }
func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string) error { func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string) error {
started := time.Now()
c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url) c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url)
vc := vcenter.New(c.Logger, c.VcCreds) vc := vcenter.New(c.Logger, c.VcCreds)
if err := vc.Login(url); err != nil { if err := vc.Login(url); err != nil {
metrics.RecordVcenterSnapshot(url, time.Since(started), 0, err)
return fmt.Errorf("unable to connect to vcenter: %w", err) return fmt.Errorf("unable to connect to vcenter: %w", err)
} }
defer func() { defer func() {
@@ -740,6 +706,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
vcVms, err := vc.GetAllVMsWithProps() vcVms, err := vc.GetAllVMsWithProps()
if err != nil { if err != nil {
metrics.RecordVcenterSnapshot(url, time.Since(started), 0, err)
return fmt.Errorf("unable to get VMs from vcenter: %w", err) return fmt.Errorf("unable to get VMs from vcenter: %w", err)
} }
canDetectMissing := len(vcVms) > 0 canDetectMissing := len(vcVms) > 0
@@ -856,6 +823,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
} }
if err := insertHourlyBatch(ctx, dbConn, tableName, batch); err != nil { if err := insertHourlyBatch(ctx, dbConn, tableName, batch); err != nil {
metrics.RecordVcenterSnapshot(url, time.Since(started), totals.VmCount, err)
return err return err
} }
@@ -866,6 +834,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
"ram_total_gb", totals.RamTotal, "ram_total_gb", totals.RamTotal,
"disk_total_gb", totals.DiskTotal, "disk_total_gb", totals.DiskTotal,
) )
metrics.RecordVcenterSnapshot(url, time.Since(started), totals.VmCount, nil)
return nil return nil
} }

View File

@@ -6,6 +6,7 @@ import (
"log/slog" "log/slog"
"time" "time"
"vctp/db" "vctp/db"
"vctp/internal/metrics"
"vctp/internal/report" "vctp/internal/report"
) )
@@ -29,6 +30,7 @@ func (c *CronTask) AggregateMonthlySummary(ctx context.Context, month time.Time,
} }
func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time.Time, force bool) error { func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time.Time, force bool) error {
jobStart := time.Now()
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
return err return err
} }
@@ -107,9 +109,12 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
if err := c.generateReport(ctx, monthlyTable); err != nil { if err := c.generateReport(ctx, monthlyTable); err != nil {
c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable) c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable)
metrics.RecordMonthlyAggregation(time.Since(jobStart), err)
return err
} }
c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable) c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable)
metrics.RecordMonthlyAggregation(time.Since(jobStart), nil)
return nil return nil
} }

17
server/handler/metrics.go Normal file
View File

@@ -0,0 +1,17 @@
package handler
import (
"net/http"
"vctp/internal/metrics"
)
// Metrics exposes Prometheus metrics.
// @Summary Prometheus metrics
// @Description Exposes Prometheus metrics for vctp.
// @Tags metrics
// @Produce plain
// @Success 200 "Prometheus metrics"
// @Router /metrics [get]
func (h *Handler) Metrics(w http.ResponseWriter, r *http.Request) {
metrics.Handler().ServeHTTP(w, r)
}

View File

@@ -65,6 +65,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce) mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce)
mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate) mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate)
mux.HandleFunc("/api/snapshots/regenerate-hourly-reports", h.SnapshotRegenerateHourlyReports) mux.HandleFunc("/api/snapshots/regenerate-hourly-reports", h.SnapshotRegenerateHourlyReports)
mux.HandleFunc("/metrics", h.Metrics)
mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList) mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList)
mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList) mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList)