more index cleanups to optimise space

db/helpers.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log/slog"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -430,6 +431,16 @@ ORDER BY tablename DESC
 
 // CleanupHourlySnapshotIndexes drops low-value per-table indexes on hourly snapshot tables.
 func CleanupHourlySnapshotIndexes(ctx context.Context, dbConn *sqlx.DB) (int, error) {
+	return cleanupHourlySnapshotIndexes(ctx, dbConn, 0)
+}
+
+// CleanupHourlySnapshotIndexesOlderThan drops per-table hourly indexes for snapshot tables older than cutoff.
+// cutoff <= 0 means drop across all hourly tables.
+func CleanupHourlySnapshotIndexesOlderThan(ctx context.Context, dbConn *sqlx.DB, cutoff time.Time) (int, error) {
+	return cleanupHourlySnapshotIndexes(ctx, dbConn, cutoff.Unix())
+}
+
+func cleanupHourlySnapshotIndexes(ctx context.Context, dbConn *sqlx.DB, cutoffUnix int64) (int, error) {
 	driver := strings.ToLower(dbConn.DriverName())
 	if driver != "sqlite" {
 		return 0, fmt.Errorf("hourly snapshot index cleanup is only supported for sqlite")
@@ -438,25 +449,68 @@ func CleanupHourlySnapshotIndexes(ctx context.Context, dbConn *sqlx.DB) (int, er
 	if err != nil {
 		return 0, err
 	}
+
+	var existing []struct {
+		Name string `db:"name"`
+	}
+	if err := selectLog(ctx, dbConn, &existing, `
+SELECT name
+FROM sqlite_master
+WHERE type = 'index'
+  AND tbl_name LIKE 'inventory_hourly_%'
+`); err != nil {
+		return 0, err
+	}
+	existingSet := make(map[string]struct{}, len(existing))
+	for _, idx := range existing {
+		existingSet[idx.Name] = struct{}{}
+	}
+
 	dropped := 0
 	for _, tableName := range tables {
 		if _, err := SafeTableName(tableName); err != nil {
 			continue
 		}
-		indexes := []string{
-			fmt.Sprintf(`DROP INDEX IF EXISTS %s_snapshottime_idx`, tableName),
-			fmt.Sprintf(`DROP INDEX IF EXISTS %s_resourcepool_idx`, tableName),
+		snapshotUnix, ok := parseHourlySnapshotUnix(tableName)
+		if !ok {
+			continue
 		}
-		for _, stmt := range indexes {
-			if _, err := execLog(ctx, dbConn, stmt); err != nil {
+		if cutoffUnix > 0 && snapshotUnix >= cutoffUnix {
+			continue
+		}
+
+		indexNames := []string{
+			fmt.Sprintf("%s_vm_vcenter_idx", tableName),
+			fmt.Sprintf("%s_snapshottime_idx", tableName),
+			fmt.Sprintf("%s_resourcepool_idx", tableName),
+		}
+		for _, indexName := range indexNames {
+			if _, exists := existingSet[indexName]; !exists {
+				continue
+			}
+			if _, err := execLog(ctx, dbConn, fmt.Sprintf(`DROP INDEX IF EXISTS %s`, indexName)); err != nil {
 				return dropped, err
 			}
+			delete(existingSet, indexName)
 			dropped++
 		}
 	}
 	return dropped, nil
 }
 
+func parseHourlySnapshotUnix(tableName string) (int64, bool) {
+	const prefix = "inventory_hourly_"
+	if !strings.HasPrefix(tableName, prefix) {
+		return 0, false
+	}
+	suffix := strings.TrimPrefix(tableName, prefix)
+	unix, err := strconv.ParseInt(suffix, 10, 64)
+	if err != nil || unix <= 0 {
+		return 0, false
+	}
+	return unix, true
+}
+
 // BackfillSerialColumn sets missing values in a serial-like column for Postgres tables.
 func BackfillSerialColumn(ctx context.Context, dbConn *sqlx.DB, tableName, columnName string) error {
 	if err := ValidateTableName(tableName); err != nil {
@@ -550,6 +604,7 @@ CREATE TABLE IF NOT EXISTS vm_hourly_stats (
 		return err
 	}
 	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_hourly_stats_vmuuid_time_idx ON vm_hourly_stats ("VmUuid","SnapshotTime")`)
+	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_hourly_stats_vmid_time_idx ON vm_hourly_stats ("VmId","SnapshotTime")`)
 	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_hourly_stats_snapshottime_idx ON vm_hourly_stats ("SnapshotTime")`)
 	return nil
 }
@@ -1243,8 +1298,38 @@ type VmLifecycle struct {
 }
 
 // FetchVmTrace returns combined hourly snapshot records for a VM (by id/uuid/name) ordered by snapshot time.
-// To avoid SQLite's UNION term limits, this iterates tables one by one and merges in-memory.
+// It prefers the shared vm_hourly_stats history table and falls back to per-snapshot tables.
 func FetchVmTrace(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) ([]VmTraceRow, error) {
+	if TableExists(ctx, dbConn, "vm_hourly_stats") {
+		rows, err := fetchVmTraceFromHourlyCache(ctx, dbConn, vmID, vmUUID, name)
+		if err != nil {
+			slog.Warn("vm trace cache query failed; falling back to hourly tables", "error", err)
+		} else if len(rows) > 0 {
+			slog.Debug("vm trace loaded from hourly cache", "row_count", len(rows))
+			return rows, nil
+		}
+	}
+	return fetchVmTraceFromSnapshotTables(ctx, dbConn, vmID, vmUUID, name)
+}
+
+func fetchVmTraceFromHourlyCache(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) ([]VmTraceRow, error) {
+	query := `
+SELECT "SnapshotTime","Name","Vcenter","VmId","VmUuid","ResourcePool","VcpuCount","RamGB","ProvisionedDisk",
+       COALESCE("CreationTime",0) AS "CreationTime",
+       COALESCE("DeletionTime",0) AS "DeletionTime"
+FROM vm_hourly_stats
+WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
+ORDER BY "SnapshotTime"
+`
+	query = dbConn.Rebind(query)
+	var rows []VmTraceRow
+	if err := selectLog(ctx, dbConn, &rows, query, vmID, vmUUID, name); err != nil {
+		return nil, err
+	}
+	return rows, nil
+}
+
+func fetchVmTraceFromSnapshotTables(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) ([]VmTraceRow, error) {
 	var tables []struct {
 		TableName    string `db:"table_name"`
 		SnapshotTime int64  `db:"snapshot_time"`
@@ -1281,7 +1366,6 @@ WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
 `, t.SnapshotTime, t.TableName)
 		args := []interface{}{vmID, vmUUID, name}
 		if driver != "sqlite" {
-			// convert ? to $1 style for postgres/pgx
 			query = strings.Replace(query, "?", "$1", 1)
 			query = strings.Replace(query, "?", "$2", 1)
 			query = strings.Replace(query, "?", "$3", 1)
@@ -1302,8 +1386,134 @@ WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
 	return rows, nil
 }
 
-// FetchVmLifecycle walks hourly snapshots to determine lifecycle bounds for a VM.
+// FetchVmLifecycle walks VM history data to determine lifecycle bounds for a VM.
+// It prefers vm_hourly_stats + vm_lifecycle_cache and falls back to per-snapshot table probes.
 func FetchVmLifecycle(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) (VmLifecycle, error) {
+	if TableExists(ctx, dbConn, "vm_hourly_stats") {
+		lifecycle, found, err := fetchVmLifecycleFromHourlyCache(ctx, dbConn, vmID, vmUUID, name)
+		if err != nil {
+			slog.Warn("vm lifecycle cache query failed; falling back to hourly tables", "error", err)
+		} else if found {
+			if TableExists(ctx, dbConn, "vm_lifecycle_cache") {
+				cached, cachedFound, cacheErr := fetchVmLifecycleFromLifecycleCache(ctx, dbConn, vmID, vmUUID, name)
+				if cacheErr != nil {
+					slog.Warn("vm lifecycle cache lookup failed", "error", cacheErr)
+				} else if cachedFound {
+					lifecycle = mergeVmLifecycle(lifecycle, cached)
+				}
+			}
+			return lifecycle, nil
+		}
+	}
+	return fetchVmLifecycleFromSnapshotTables(ctx, dbConn, vmID, vmUUID, name)
+}
+
+func fetchVmLifecycleFromHourlyCache(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) (VmLifecycle, bool, error) {
+	var row struct {
+		Rows      int64         `db:"rows"`
+		Creation  sql.NullInt64 `db:"creation_time"`
+		FirstSeen sql.NullInt64 `db:"first_seen"`
+		LastSeen  sql.NullInt64 `db:"last_seen"`
+		Deletion  sql.NullInt64 `db:"deletion_time"`
+	}
+	query := `
+SELECT
+  COUNT(1) AS rows,
+  MIN(NULLIF("CreationTime",0)) AS creation_time,
+  MIN("SnapshotTime") AS first_seen,
+  MAX("SnapshotTime") AS last_seen,
+  MIN(NULLIF("DeletionTime",0)) AS deletion_time
+FROM vm_hourly_stats
+WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
+`
+	query = dbConn.Rebind(query)
+	if err := getLog(ctx, dbConn, &row, query, vmID, vmUUID, name); err != nil {
+		return VmLifecycle{}, false, err
+	}
+	if row.Rows == 0 {
+		return VmLifecycle{}, false, nil
+	}
+	lifecycle := VmLifecycle{
+		FirstSeen: row.FirstSeen.Int64,
+		LastSeen:  row.LastSeen.Int64,
+	}
+	if row.Creation.Valid && row.Creation.Int64 > 0 {
+		lifecycle.CreationTime = row.Creation.Int64
+		lifecycle.CreationApprox = false
+	} else if row.FirstSeen.Valid && row.FirstSeen.Int64 > 0 {
+		lifecycle.CreationTime = row.FirstSeen.Int64
+		lifecycle.CreationApprox = true
+	}
+	if row.Deletion.Valid && row.Deletion.Int64 > 0 {
+		lifecycle.DeletionTime = row.Deletion.Int64
+	}
+	return lifecycle, true, nil
+}
+
+func fetchVmLifecycleFromLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) (VmLifecycle, bool, error) {
+	var row struct {
+		Rows      int64         `db:"rows"`
+		FirstSeen sql.NullInt64 `db:"first_seen"`
+		LastSeen  sql.NullInt64 `db:"last_seen"`
+		Deletion  sql.NullInt64 `db:"deletion_time"`
+	}
+	query := `
+SELECT
+  COUNT(1) AS rows,
+  MIN(NULLIF("FirstSeen",0)) AS first_seen,
+  MAX(NULLIF("LastSeen",0)) AS last_seen,
+  MIN(NULLIF("DeletedAt",0)) AS deletion_time
+FROM vm_lifecycle_cache
+WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
+`
+	query = dbConn.Rebind(query)
+	if err := getLog(ctx, dbConn, &row, query, vmID, vmUUID, name); err != nil {
+		return VmLifecycle{}, false, err
+	}
+	if row.Rows == 0 {
+		return VmLifecycle{}, false, nil
+	}
+	lifecycle := VmLifecycle{
+		FirstSeen: row.FirstSeen.Int64,
+		LastSeen:  row.LastSeen.Int64,
+	}
+	if row.FirstSeen.Valid && row.FirstSeen.Int64 > 0 {
+		lifecycle.CreationTime = row.FirstSeen.Int64
+		lifecycle.CreationApprox = true
+	}
+	if row.Deletion.Valid && row.Deletion.Int64 > 0 {
+		lifecycle.DeletionTime = row.Deletion.Int64
+	}
+	return lifecycle, true, nil
+}
+
+func mergeVmLifecycle(base, overlay VmLifecycle) VmLifecycle {
+	out := base
+	if overlay.FirstSeen > 0 && (out.FirstSeen == 0 || overlay.FirstSeen < out.FirstSeen) {
+		out.FirstSeen = overlay.FirstSeen
+	}
+	if overlay.LastSeen > out.LastSeen {
+		out.LastSeen = overlay.LastSeen
+	}
+	if overlay.DeletionTime > 0 && (out.DeletionTime == 0 || overlay.DeletionTime < out.DeletionTime) {
+		out.DeletionTime = overlay.DeletionTime
+	}
+	if overlay.CreationTime > 0 {
+		if out.CreationTime == 0 || overlay.CreationTime < out.CreationTime {
+			out.CreationTime = overlay.CreationTime
+			out.CreationApprox = overlay.CreationApprox
+		} else if out.CreationTime == overlay.CreationTime && !overlay.CreationApprox {
+			out.CreationApprox = false
+		}
+	}
+	if out.CreationTime == 0 && out.FirstSeen > 0 {
+		out.CreationTime = out.FirstSeen
+		out.CreationApprox = true
+	}
+	return out
+}
+
+func fetchVmLifecycleFromSnapshotTables(ctx context.Context, dbConn *sqlx.DB, vmID, vmUUID, name string) (VmLifecycle, error) {
 	var lifecycle VmLifecycle
 	var tables []struct {
 		TableName    string `db:"table_name"`
@@ -1325,7 +1535,6 @@ ORDER BY snapshot_time
 		if err := ValidateTableName(t.TableName); err != nil {
 			continue
 		}
-		// Probe this table for the VM.
 		query := fmt.Sprintf(`
 SELECT MIN(NULLIF("CreationTime",0)) AS min_creation, COUNT(1) AS cnt
 FROM %s
@@ -1362,7 +1571,6 @@ WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
 				break
 			}
 		} else {
-			// reset if we haven't seen the VM yet
 			consecutiveMissing = 0
 		}
 	}