avoid vcenter totals pages scanning whole database

2026-02-09 13:44:43 +11:00
parent c66679a71f
commit 5736dc6929
11 changed files with 991 additions and 195 deletions
--- a/db/helpers.go
+++ b/db/helpers.go
@@ -1121,6 +1121,277 @@ CREATE TABLE IF NOT EXISTS vcenter_totals (
 	return nil
 }

+// EnsureVcenterLatestTotalsTable creates a compact table with one latest totals row per vCenter.
+func EnsureVcenterLatestTotalsTable(ctx context.Context, dbConn *sqlx.DB) error {
+	driver := strings.ToLower(dbConn.DriverName())
+	var ddl string
+	switch driver {
+	case "pgx", "postgres":
+		ddl = `
+CREATE TABLE IF NOT EXISTS vcenter_latest_totals (
+  "Vcenter" TEXT PRIMARY KEY,
+  "SnapshotTime" BIGINT NOT NULL,
+  "VmCount" BIGINT NOT NULL,
+  "VcpuTotal" BIGINT NOT NULL,
+  "RamTotalGB" BIGINT NOT NULL
+);`
+	default:
+		ddl = `
+CREATE TABLE IF NOT EXISTS vcenter_latest_totals (
+  "Vcenter" TEXT PRIMARY KEY,
+  "SnapshotTime" BIGINT NOT NULL,
+  "VmCount" BIGINT NOT NULL,
+  "VcpuTotal" BIGINT NOT NULL,
+  "RamTotalGB" BIGINT NOT NULL
+);`
+	}
+	if _, err := execLog(ctx, dbConn, ddl); err != nil {
+		return err
+	}
+	return nil
+}
+
+// UpsertVcenterLatestTotals stores the latest totals per vCenter.
+func UpsertVcenterLatestTotals(ctx context.Context, dbConn *sqlx.DB, vcenter string, snapshotTime int64, vmCount, vcpuTotal, ramTotal int64) error {
+	if strings.TrimSpace(vcenter) == "" {
+		return fmt.Errorf("vcenter is empty")
+	}
+	if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
+		return err
+	}
+	_, err := execLog(ctx, dbConn, `
+INSERT INTO vcenter_latest_totals ("Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB")
+VALUES ($1,$2,$3,$4,$5)
+ON CONFLICT ("Vcenter") DO UPDATE SET
+  "SnapshotTime" = EXCLUDED."SnapshotTime",
+  "VmCount" = EXCLUDED."VmCount",
+  "VcpuTotal" = EXCLUDED."VcpuTotal",
+  "RamTotalGB" = EXCLUDED."RamTotalGB"
+WHERE EXCLUDED."SnapshotTime" >= vcenter_latest_totals."SnapshotTime"
+`, vcenter, snapshotTime, vmCount, vcpuTotal, ramTotal)
+	return err
+}
+
+// EnsureVcenterAggregateTotalsTable creates a compact cache for hourly/daily (and optional monthly) totals.
+func EnsureVcenterAggregateTotalsTable(ctx context.Context, dbConn *sqlx.DB) error {
+	ddl := `
+CREATE TABLE IF NOT EXISTS vcenter_aggregate_totals (
+  "SnapshotType" TEXT NOT NULL,
+  "Vcenter" TEXT NOT NULL,
+  "SnapshotTime" BIGINT NOT NULL,
+  "VmCount" BIGINT NOT NULL,
+  "VcpuTotal" BIGINT NOT NULL,
+  "RamTotalGB" BIGINT NOT NULL,
+  PRIMARY KEY ("SnapshotType","Vcenter","SnapshotTime")
+);`
+	if _, err := execLog(ctx, dbConn, ddl); err != nil {
+		return err
+	}
+	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vcenter_aggregate_totals_vc_type_time_idx ON vcenter_aggregate_totals ("Vcenter","SnapshotType","SnapshotTime" DESC)`)
+	return nil
+}
+
+// UpsertVcenterAggregateTotal stores per-vCenter totals for a snapshot type/time.
+func UpsertVcenterAggregateTotal(ctx context.Context, dbConn *sqlx.DB, snapshotType, vcenter string, snapshotTime int64, vmCount, vcpuTotal, ramTotal int64) error {
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
+	if snapshotType == "" {
+		return fmt.Errorf("snapshot type is empty")
+	}
+	if strings.TrimSpace(vcenter) == "" {
+		return fmt.Errorf("vcenter is empty")
+	}
+	if err := EnsureVcenterAggregateTotalsTable(ctx, dbConn); err != nil {
+		return err
+	}
+	_, err := execLog(ctx, dbConn, `
+INSERT INTO vcenter_aggregate_totals ("SnapshotType","Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB")
+VALUES ($1,$2,$3,$4,$5,$6)
+ON CONFLICT ("SnapshotType","Vcenter","SnapshotTime") DO UPDATE SET
+  "VmCount" = EXCLUDED."VmCount",
+  "VcpuTotal" = EXCLUDED."VcpuTotal",
+  "RamTotalGB" = EXCLUDED."RamTotalGB"
+`, snapshotType, vcenter, snapshotTime, vmCount, vcpuTotal, ramTotal)
+	return err
+}
+
+// ReplaceVcenterAggregateTotalsFromSummary recomputes one snapshot's per-vCenter totals from a summary table.
+func ReplaceVcenterAggregateTotalsFromSummary(ctx context.Context, dbConn *sqlx.DB, summaryTable, snapshotType string, snapshotTime int64) (int, error) {
+	if err := ValidateTableName(summaryTable); err != nil {
+		return 0, err
+	}
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
+	if snapshotType == "" {
+		return 0, fmt.Errorf("snapshot type is empty")
+	}
+	if err := EnsureVcenterAggregateTotalsTable(ctx, dbConn); err != nil {
+		return 0, err
+	}
+	if _, err := execLog(ctx, dbConn, `
+DELETE FROM vcenter_aggregate_totals
+WHERE "SnapshotType" = $1 AND "SnapshotTime" = $2
+`, snapshotType, snapshotTime); err != nil {
+		return 0, err
+	}
+	query := fmt.Sprintf(`
+SELECT
+  "Vcenter" AS vcenter,
+  COUNT(1) AS vm_count,
+  CAST(COALESCE(SUM(COALESCE("AvgVcpuCount","VcpuCount")),0) AS BIGINT) AS vcpu_total,
+  CAST(COALESCE(SUM(COALESCE("AvgRamGB","RamGB")),0) AS BIGINT) AS ram_total
+FROM %s
+GROUP BY "Vcenter"
+`, summaryTable)
+	var rows []struct {
+		Vcenter   string `db:"vcenter"`
+		VmCount   int64  `db:"vm_count"`
+		VcpuTotal int64  `db:"vcpu_total"`
+		RamTotal  int64  `db:"ram_total"`
+	}
+	if err := selectLog(ctx, dbConn, &rows, query); err != nil {
+		return 0, err
+	}
+	upserted := 0
+	for _, row := range rows {
+		if err := UpsertVcenterAggregateTotal(ctx, dbConn, snapshotType, row.Vcenter, snapshotTime, row.VmCount, row.VcpuTotal, row.RamTotal); err != nil {
+			return upserted, err
+		}
+		upserted++
+	}
+	return upserted, nil
+}
+
+// SyncVcenterAggregateTotalsFromRegistry refreshes cached totals for summary snapshots listed in snapshot_registry.
+func SyncVcenterAggregateTotalsFromRegistry(ctx context.Context, dbConn *sqlx.DB, snapshotType string) (int, int, error) {
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
+	if snapshotType == "" {
+		return 0, 0, fmt.Errorf("snapshot type is empty")
+	}
+	if snapshotType != "daily" && snapshotType != "monthly" {
+		return 0, 0, fmt.Errorf("unsupported snapshot type for summary cache sync: %s", snapshotType)
+	}
+	if err := EnsureVcenterAggregateTotalsTable(ctx, dbConn); err != nil {
+		return 0, 0, err
+	}
+
+	query := dbConn.Rebind(`
+SELECT table_name, snapshot_time
+FROM snapshot_registry
+WHERE snapshot_type = ?
+ORDER BY snapshot_time
+`)
+	var snapshots []struct {
+		TableName    string `db:"table_name"`
+		SnapshotTime int64  `db:"snapshot_time"`
+	}
+	if err := selectLog(ctx, dbConn, &snapshots, query, snapshotType); err != nil {
+		return 0, 0, err
+	}
+
+	snapshotsRefreshed := 0
+	rowsUpserted := 0
+	failures := 0
+	for _, snapshot := range snapshots {
+		if err := ValidateTableName(snapshot.TableName); err != nil {
+			failures++
+			slog.Warn("skipping invalid summary table in snapshot registry", "snapshot_type", snapshotType, "table", snapshot.TableName, "error", err)
+			continue
+		}
+		upserted, err := ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, snapshot.TableName, snapshotType, snapshot.SnapshotTime)
+		if err != nil {
+			failures++
+			slog.Warn("failed to refresh vcenter aggregate cache from summary table", "snapshot_type", snapshotType, "table", snapshot.TableName, "snapshot_time", snapshot.SnapshotTime, "error", err)
+			continue
+		}
+		snapshotsRefreshed++
+		rowsUpserted += upserted
+	}
+
+	if failures > 0 {
+		return snapshotsRefreshed, rowsUpserted, fmt.Errorf("vcenter aggregate cache sync finished with %d failed snapshot(s)", failures)
+	}
+	return snapshotsRefreshed, rowsUpserted, nil
+}
+
+// ListVcenterAggregateTotals lists cached totals by type.
+func ListVcenterAggregateTotals(ctx context.Context, dbConn *sqlx.DB, vcenter, snapshotType string, limit int) ([]VcenterTotalRow, error) {
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
+	if err := EnsureVcenterAggregateTotalsTable(ctx, dbConn); err != nil {
+		return nil, err
+	}
+	if limit <= 0 {
+		limit = 200
+	}
+	rows := make([]VcenterTotalRow, 0, limit)
+	query := `
+SELECT "Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB"
+FROM vcenter_aggregate_totals
+WHERE "Vcenter" = $1 AND "SnapshotType" = $2
+ORDER BY "SnapshotTime" DESC
+LIMIT $3
+`
+	if err := selectLog(ctx, dbConn, &rows, query, vcenter, snapshotType, limit); err != nil {
+		return nil, err
+	}
+	return rows, nil
+}
+
+// ListVcenterAggregateTotalsSince lists cached totals by type from a lower-bound timestamp.
+func ListVcenterAggregateTotalsSince(ctx context.Context, dbConn *sqlx.DB, vcenter, snapshotType string, since time.Time) ([]VcenterTotalRow, error) {
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
+	if err := EnsureVcenterAggregateTotalsTable(ctx, dbConn); err != nil {
+		return nil, err
+	}
+	rows := make([]VcenterTotalRow, 0, 256)
+	query := `
+SELECT "Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB"
+FROM vcenter_aggregate_totals
+WHERE "Vcenter" = $1 AND "SnapshotType" = $2 AND "SnapshotTime" >= $3
+ORDER BY "SnapshotTime" DESC
+`
+	if err := selectLog(ctx, dbConn, &rows, query, vcenter, snapshotType, since.Unix()); err != nil {
+		return nil, err
+	}
+	return rows, nil
+}
+
+// SyncVcenterLatestTotalsFromHistory backfills latest totals from existing vcenter_totals history.
+func SyncVcenterLatestTotalsFromHistory(ctx context.Context, dbConn *sqlx.DB) (int, error) {
+	if err := EnsureVcenterTotalsTable(ctx, dbConn); err != nil {
+		return 0, err
+	}
+	if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
+		return 0, err
+	}
+	var rows []struct {
+		Vcenter      string `db:"Vcenter"`
+		SnapshotTime int64  `db:"SnapshotTime"`
+		VmCount      int64  `db:"VmCount"`
+		VcpuTotal    int64  `db:"VcpuTotal"`
+		RamTotalGB   int64  `db:"RamTotalGB"`
+	}
+	if err := selectLog(ctx, dbConn, &rows, `
+SELECT t."Vcenter", t."SnapshotTime", t."VmCount", t."VcpuTotal", t."RamTotalGB"
+FROM vcenter_totals t
+JOIN (
+  SELECT "Vcenter", MAX("SnapshotTime") AS max_snapshot_time
+  FROM vcenter_totals
+  GROUP BY "Vcenter"
+) latest
+  ON latest."Vcenter" = t."Vcenter"
+ AND latest.max_snapshot_time = t."SnapshotTime"
+`); err != nil {
+		return 0, err
+	}
+	upserted := 0
+	for _, row := range rows {
+		if err := UpsertVcenterLatestTotals(ctx, dbConn, row.Vcenter, row.SnapshotTime, row.VmCount, row.VcpuTotal, row.RamTotalGB); err != nil {
+			return upserted, err
+		}
+		upserted++
+	}
+	return upserted, nil
+}
+
 // InsertVcenterTotals records totals for a vcenter at a snapshot time.
 func InsertVcenterTotals(ctx context.Context, dbConn *sqlx.DB, vcenter string, snapshotTime time.Time, vmCount, vcpuTotal, ramTotal int64) error {
 	if strings.TrimSpace(vcenter) == "" {
@@ -1129,15 +1400,56 @@ func InsertVcenterTotals(ctx context.Context, dbConn *sqlx.DB, vcenter string, s
 	if err := EnsureVcenterTotalsTable(ctx, dbConn); err != nil {
 		return err
 	}
+	if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
+		return err
+	}
 	_, err := execLog(ctx, dbConn, `
 INSERT INTO vcenter_totals ("Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB")
 VALUES ($1,$2,$3,$4,$5)
 `, vcenter, snapshotTime.Unix(), vmCount, vcpuTotal, ramTotal)
-	return err
+	if err != nil {
+		return err
+	}
+	if err := UpsertVcenterLatestTotals(ctx, dbConn, vcenter, snapshotTime.Unix(), vmCount, vcpuTotal, ramTotal); err != nil {
+		return err
+	}
+	if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", vcenter, snapshotTime.Unix(), vmCount, vcpuTotal, ramTotal); err != nil {
+		slog.Warn("failed to upsert vcenter_aggregate_totals", "snapshot_type", "hourly", "vcenter", vcenter, "snapshot_time", snapshotTime.Unix(), "error", err)
+	}
+	return nil
 }

 // ListVcenters returns distinct vcenter URLs tracked.
 func ListVcenters(ctx context.Context, dbConn *sqlx.DB) ([]string, error) {
+	if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err == nil {
+		rows, err := dbConn.QueryxContext(ctx, `SELECT "Vcenter" FROM vcenter_latest_totals ORDER BY "Vcenter"`)
+		if err == nil {
+			defer rows.Close()
+			out := make([]string, 0, 32)
+			for rows.Next() {
+				var v string
+				if err := rows.Scan(&v); err != nil {
+					return nil, err
+				}
+				out = append(out, v)
+			}
+			if err := rows.Err(); err != nil {
+				return nil, err
+			}
+			if len(out) > 0 {
+				return out, nil
+			}
+			// Older installs may have vcenter_totals populated but no latest cache yet.
+			if _, err := SyncVcenterLatestTotalsFromHistory(ctx, dbConn); err == nil {
+				refreshed := make([]string, 0, 32)
+				if err := selectLog(ctx, dbConn, &refreshed, `SELECT "Vcenter" FROM vcenter_latest_totals ORDER BY "Vcenter"`); err == nil && len(refreshed) > 0 {
+					return refreshed, nil
+				}
+			}
+		}
+	}
+
+	// Fallback for older DBs before vcenter_latest_totals gets populated.
 	if err := EnsureVcenterTotalsTable(ctx, dbConn); err != nil {
 		return nil, err
 	}
@@ -1187,15 +1499,66 @@ LIMIT $2`
 	return rows, nil
 }

+// ListVcenterHourlyTotalsSince returns hourly totals for a vCenter from a minimum snapshot time.
+func ListVcenterHourlyTotalsSince(ctx context.Context, dbConn *sqlx.DB, vcenter string, since time.Time) ([]VcenterTotalRow, error) {
+	cachedRows, cacheErr := ListVcenterAggregateTotalsSince(ctx, dbConn, vcenter, "hourly", since)
+	if cacheErr == nil && len(cachedRows) > 0 {
+		return cachedRows, nil
+	}
+	if cacheErr != nil {
+		slog.Warn("failed to read hourly totals cache", "vcenter", vcenter, "since", since.Unix(), "error", cacheErr)
+	}
+
+	if err := EnsureVcenterTotalsTable(ctx, dbConn); err != nil {
+		return nil, err
+	}
+	rows := make([]VcenterTotalRow, 0, 256)
+	query := `
+SELECT "Vcenter","SnapshotTime","VmCount","VcpuTotal","RamTotalGB"
+FROM vcenter_totals
+WHERE "Vcenter" = $1
+  AND "SnapshotTime" >= $2
+ORDER BY "SnapshotTime" DESC`
+	if err := selectLog(ctx, dbConn, &rows, query, vcenter, since.Unix()); err != nil {
+		return nil, err
+	}
+	for _, row := range rows {
+		if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", row.Vcenter, row.SnapshotTime, row.VmCount, row.VcpuTotal, row.RamTotalGB); err != nil {
+			slog.Warn("failed to warm hourly totals cache", "vcenter", row.Vcenter, "snapshot_time", row.SnapshotTime, "error", err)
+			break
+		}
+	}
+	return rows, nil
+}
+
 // ListVcenterTotalsByType returns totals for a vcenter for the requested snapshot type (hourly, daily, monthly).
-// Hourly values come from vcenter_totals; daily/monthly are derived from the summary tables referenced in snapshot_registry.
+// Prefer vcenter_aggregate_totals cache and fallback to source tables when cache is empty.
 func ListVcenterTotalsByType(ctx context.Context, dbConn *sqlx.DB, vcenter string, snapshotType string, limit int) ([]VcenterTotalRow, error) {
-	snapshotType = strings.ToLower(snapshotType)
+	snapshotType = strings.ToLower(strings.TrimSpace(snapshotType))
 	if snapshotType == "" {
 		snapshotType = "hourly"
 	}
+
+	cachedRows, cacheErr := ListVcenterAggregateTotals(ctx, dbConn, vcenter, snapshotType, limit)
+	if cacheErr == nil && len(cachedRows) > 0 {
+		return cachedRows, nil
+	}
+	if cacheErr != nil {
+		slog.Warn("failed to read vcenter aggregate totals cache", "snapshot_type", snapshotType, "vcenter", vcenter, "error", cacheErr)
+	}
+
 	if snapshotType == "hourly" {
-		return ListVcenterTotals(ctx, dbConn, vcenter, limit)
+		rows, err := ListVcenterTotals(ctx, dbConn, vcenter, limit)
+		if err != nil {
+			return nil, err
+		}
+		for _, row := range rows {
+			if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", row.Vcenter, row.SnapshotTime, row.VmCount, row.VcpuTotal, row.RamTotalGB); err != nil {
+				slog.Warn("failed to warm hourly totals cache", "vcenter", row.Vcenter, "snapshot_time", row.SnapshotTime, "error", err)
+				break
+			}
+		}
+		return rows, nil
 	}

 	if limit <= 0 {
@@ -1240,6 +1603,12 @@ LIMIT $2
 			RamTotalGB:   agg.RamTotalGB,
 		})
 	}
+	for _, row := range out {
+		if err := UpsertVcenterAggregateTotal(ctx, dbConn, snapshotType, row.Vcenter, row.SnapshotTime, row.VmCount, row.VcpuTotal, row.RamTotalGB); err != nil {
+			slog.Warn("failed to warm vcenter aggregate totals cache", "snapshot_type", snapshotType, "vcenter", row.Vcenter, "snapshot_time", row.SnapshotTime, "error", err)
+			break
+		}
+	}
 	return out, nil
 }

@@ -1589,6 +1958,9 @@ func SyncVcenterTotalsFromSnapshots(ctx context.Context, dbConn *sqlx.DB) error
 	if err := EnsureVcenterTotalsTable(ctx, dbConn); err != nil {
 		return err
 	}
+	if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
+		return err
+	}
 	driver := strings.ToLower(dbConn.DriverName())
 	var hourlyTables []struct {
 		TableName    string `db:"table_name"`
@@ -1640,6 +2012,12 @@ WHERE NOT EXISTS (
 			if _, err := execLog(ctx, dbConn, insert, a.Vcenter, ht.SnapshotTime, a.VmCount, a.VcpuTotal, a.RamTotal); err != nil {
 				slog.Warn("failed to backfill vcenter_totals", "table", ht.TableName, "vcenter", a.Vcenter, "snapshot_time", ht.SnapshotTime, "error", err)
 			}
+			if err := UpsertVcenterLatestTotals(ctx, dbConn, a.Vcenter, ht.SnapshotTime, a.VmCount, a.VcpuTotal, a.RamTotal); err != nil {
+				slog.Warn("failed to upsert vcenter_latest_totals", "table", ht.TableName, "vcenter", a.Vcenter, "snapshot_time", ht.SnapshotTime, "error", err)
+			}
+			if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", a.Vcenter, ht.SnapshotTime, a.VmCount, a.VcpuTotal, a.RamTotal); err != nil {
+				slog.Warn("failed to upsert vcenter_aggregate_totals", "table", ht.TableName, "snapshot_type", "hourly", "vcenter", a.Vcenter, "snapshot_time", ht.SnapshotTime, "error", err)
+			}
 		}
 	}
 	return nil