Implement targeted VM property refresh and backfill logic for snapshot rows

internal/tasks/inventorySnapshots.go

@@ -690,6 +690,33 @@ func snapshotFromVM(ctx context.Context, dbConn *sqlx.DB, vmObject *mo.VirtualMa
 		return InventorySnapshotRow{}, fmt.Errorf("missing VM object")
 	}
 
+	// Rarely, bulk property retrieval can return partial VM objects (for example, nil Config).
+	// Do a targeted per-VM refresh before building the snapshot row to avoid writing sparse rows.
+	if vmObject.Config == nil || vmObject.ResourcePool == nil || vmObject.Parent == nil || vmObject.Runtime.Host == nil || vmObject.Runtime.PowerState == "" {
+		if refreshed, err := vc.GetVMWithSnapshotPropsByRef(vmObject.Reference()); err == nil && refreshed != nil {
+			if vmObject.Name == "" && refreshed.Name != "" {
+				vmObject.Name = refreshed.Name
+			}
+			if vmObject.Parent == nil && refreshed.Parent != nil {
+				vmObject.Parent = refreshed.Parent
+			}
+			if vmObject.Config == nil && refreshed.Config != nil {
+				vmObject.Config = refreshed.Config
+			}
+			if vmObject.ResourcePool == nil && refreshed.ResourcePool != nil {
+				vmObject.ResourcePool = refreshed.ResourcePool
+			}
+			if vmObject.Runtime.Host == nil && refreshed.Runtime.Host != nil {
+				vmObject.Runtime.Host = refreshed.Runtime.Host
+			}
+			if vmObject.Runtime.PowerState == "" && refreshed.Runtime.PowerState != "" {
+				vmObject.Runtime.PowerState = refreshed.Runtime.PowerState
+			}
+		} else if vc.Logger != nil {
+			vc.Logger.Warn("failed targeted VM property refresh for snapshot row", "vm_id", vmObject.Reference().Value, "error", err)
+		}
+	}
+
 	row := InventorySnapshotRow{
 		Name:         vmObject.Name,
 		Vcenter:      vc.Vurl,
@@ -903,9 +930,144 @@ func snapshotFromVM(ctx context.Context, dbConn *sqlx.DB, vmObject *mo.VirtualMa
 		}
 	}
 
+	if dbConn != nil && needsSnapshotBackfill(row) {
+		if backfillSnapshotRowFromHourlyCache(ctx, dbConn, &row) && vc.Logger != nil {
+			vc.Logger.Debug("backfilled sparse VM snapshot row from hourly cache", "vm_id", row.VmId.String, "name", row.Name, "vcenter", row.Vcenter)
+		}
+	}
+	if row.SrmPlaceholder == "" {
+		row.SrmPlaceholder = "FALSE"
+	}
+	if needsSnapshotBackfill(row) && vc.Logger != nil {
+		vc.Logger.Warn("snapshot row remains sparse after fallbacks",
+			"vm_id", row.VmId.String,
+			"name", row.Name,
+			"vcenter", row.Vcenter,
+			"has_creation_time", row.CreationTime.Valid,
+			"has_cluster", row.Cluster.Valid,
+			"has_disk", row.ProvisionedDisk.Valid,
+			"has_vcpu", row.VcpuCount.Valid,
+			"has_ram", row.RamGB.Valid,
+			"has_vm_uuid", row.VmUuid.Valid,
+		)
+	}
+
 	return row, nil
 }
 
+func needsSnapshotBackfill(row InventorySnapshotRow) bool {
+	return !row.CreationTime.Valid ||
+		!row.ProvisionedDisk.Valid ||
+		!row.VcpuCount.Valid ||
+		!row.RamGB.Valid ||
+		!row.Cluster.Valid ||
+		!row.Datacenter.Valid ||
+		row.SrmPlaceholder == "" ||
+		!row.VmUuid.Valid
+}
+
+func backfillSnapshotRowFromHourlyCache(ctx context.Context, dbConn *sqlx.DB, row *InventorySnapshotRow) bool {
+	if row == nil || dbConn == nil {
+		return false
+	}
+	if strings.TrimSpace(row.Vcenter) == "" || strings.TrimSpace(row.VmId.String) == "" {
+		return false
+	}
+	if !db.TableExists(ctx, dbConn, "vm_hourly_stats") {
+		return false
+	}
+
+	query := `
+SELECT
+  "CreationTime",
+  "ResourcePool",
+  "Datacenter",
+  "Cluster",
+  "Folder",
+  "ProvisionedDisk",
+  "VcpuCount",
+  "RamGB",
+  "IsTemplate",
+  "PoweredOn",
+  "SrmPlaceholder",
+  "VmUuid"
+FROM vm_hourly_stats
+WHERE "Vcenter" = ? AND "VmId" = ?
+ORDER BY "SnapshotTime" DESC
+LIMIT 1`
+	query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
+
+	var cached struct {
+		CreationTime    sql.NullInt64   `db:"CreationTime"`
+		ResourcePool    sql.NullString  `db:"ResourcePool"`
+		Datacenter      sql.NullString  `db:"Datacenter"`
+		Cluster         sql.NullString  `db:"Cluster"`
+		Folder          sql.NullString  `db:"Folder"`
+		ProvisionedDisk sql.NullFloat64 `db:"ProvisionedDisk"`
+		VcpuCount       sql.NullInt64   `db:"VcpuCount"`
+		RamGB           sql.NullInt64   `db:"RamGB"`
+		IsTemplate      sql.NullString  `db:"IsTemplate"`
+		PoweredOn       sql.NullString  `db:"PoweredOn"`
+		SrmPlaceholder  sql.NullString  `db:"SrmPlaceholder"`
+		VmUuid          sql.NullString  `db:"VmUuid"`
+	}
+	if err := dbConn.GetContext(ctx, &cached, query, row.Vcenter, row.VmId.String); err != nil {
+		return false
+	}
+
+	changed := false
+	if !row.CreationTime.Valid && cached.CreationTime.Valid {
+		row.CreationTime = cached.CreationTime
+		changed = true
+	}
+	if (!row.ResourcePool.Valid || row.ResourcePool.String == "") && cached.ResourcePool.Valid {
+		pool := normalizeResourcePool(cached.ResourcePool.String)
+		row.ResourcePool = sql.NullString{String: pool, Valid: pool != ""}
+		changed = true
+	}
+	if (!row.Datacenter.Valid || row.Datacenter.String == "") && cached.Datacenter.Valid {
+		row.Datacenter = cached.Datacenter
+		changed = true
+	}
+	if (!row.Cluster.Valid || row.Cluster.String == "") && cached.Cluster.Valid {
+		row.Cluster = cached.Cluster
+		changed = true
+	}
+	if (!row.Folder.Valid || row.Folder.String == "") && cached.Folder.Valid {
+		row.Folder = cached.Folder
+		changed = true
+	}
+	if !row.ProvisionedDisk.Valid && cached.ProvisionedDisk.Valid {
+		row.ProvisionedDisk = cached.ProvisionedDisk
+		changed = true
+	}
+	if !row.VcpuCount.Valid && cached.VcpuCount.Valid {
+		row.VcpuCount = cached.VcpuCount
+		changed = true
+	}
+	if !row.RamGB.Valid && cached.RamGB.Valid {
+		row.RamGB = cached.RamGB
+		changed = true
+	}
+	if row.IsTemplate == "" && cached.IsTemplate.Valid {
+		row.IsTemplate = boolStringFromInterface(cached.IsTemplate.String)
+		changed = true
+	}
+	if row.PoweredOn == "" && cached.PoweredOn.Valid {
+		row.PoweredOn = boolStringFromInterface(cached.PoweredOn.String)
+		changed = true
+	}
+	if row.SrmPlaceholder == "" && cached.SrmPlaceholder.Valid {
+		row.SrmPlaceholder = boolStringFromInterface(cached.SrmPlaceholder.String)
+		changed = true
+	}
+	if !row.VmUuid.Valid && cached.VmUuid.Valid {
+		row.VmUuid = cached.VmUuid
+		changed = true
+	}
+	return changed
+}
+
 func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) InventorySnapshotRow {
 	return InventorySnapshotRow{
 		InventoryId:     sql.NullInt64{Int64: inv.Iid, Valid: inv.Iid > 0},

internal/tasks/inventorySnapshots_test.go

@@ -0,0 +1,106 @@
+package tasks
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"vctp/db"
+
+	"github.com/jmoiron/sqlx"
+	_ "modernc.org/sqlite"
+)
+
+func newTasksTestDB(t *testing.T) *sqlx.DB {
+	t.Helper()
+	dbConn, err := sqlx.Open("sqlite", ":memory:")
+	if err != nil {
+		t.Fatalf("failed to open sqlite test db: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = dbConn.Close()
+	})
+	return dbConn
+}
+
+func TestBackfillSnapshotRowFromHourlyCache(t *testing.T) {
+	ctx := context.Background()
+	dbConn := newTasksTestDB(t)
+
+	if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil {
+		t.Fatalf("failed to ensure vm_hourly_stats: %v", err)
+	}
+
+	insertSQL := `
+INSERT INTO vm_hourly_stats (
+  "SnapshotTime","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","ResourcePool",
+  "Datacenter","Cluster","Folder","ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder"
+) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+`
+	if _, err := dbConn.ExecContext(ctx, insertSQL,
+		int64(1000), "vc-a", "vm-1", "uuid-1", "demo-vm", int64(900), int64(0), "Tin",
+		"dc-1", "cluster-1", "/Prod", 123.4, int64(4), int64(16), "FALSE", "TRUE", "FALSE",
+	); err != nil {
+		t.Fatalf("failed to insert cache row: %v", err)
+	}
+
+	row := InventorySnapshotRow{
+		Vcenter:        "vc-a",
+		VmId:           sql.NullString{String: "vm-1", Valid: true},
+		Name:           "demo-vm",
+		SnapshotTime:   2000,
+		ResourcePool:   sql.NullString{String: "Tin", Valid: true},
+		SrmPlaceholder: "",
+	}
+	if !needsSnapshotBackfill(row) {
+		t.Fatal("expected sparse row to require backfill")
+	}
+
+	changed := backfillSnapshotRowFromHourlyCache(ctx, dbConn, &row)
+	if !changed {
+		t.Fatal("expected cache backfill to update the row")
+	}
+
+	if !row.CreationTime.Valid || row.CreationTime.Int64 != 900 {
+		t.Fatalf("unexpected CreationTime after backfill: %#v", row.CreationTime)
+	}
+	if !row.Cluster.Valid || row.Cluster.String != "cluster-1" {
+		t.Fatalf("unexpected Cluster after backfill: %#v", row.Cluster)
+	}
+	if !row.Datacenter.Valid || row.Datacenter.String != "dc-1" {
+		t.Fatalf("unexpected Datacenter after backfill: %#v", row.Datacenter)
+	}
+	if !row.ProvisionedDisk.Valid || row.ProvisionedDisk.Float64 != 123.4 {
+		t.Fatalf("unexpected ProvisionedDisk after backfill: %#v", row.ProvisionedDisk)
+	}
+	if !row.VcpuCount.Valid || row.VcpuCount.Int64 != 4 {
+		t.Fatalf("unexpected VcpuCount after backfill: %#v", row.VcpuCount)
+	}
+	if !row.RamGB.Valid || row.RamGB.Int64 != 16 {
+		t.Fatalf("unexpected RamGB after backfill: %#v", row.RamGB)
+	}
+	if row.SrmPlaceholder != "FALSE" {
+		t.Fatalf("unexpected SrmPlaceholder after backfill: %q", row.SrmPlaceholder)
+	}
+	if !row.VmUuid.Valid || row.VmUuid.String != "uuid-1" {
+		t.Fatalf("unexpected VmUuid after backfill: %#v", row.VmUuid)
+	}
+}
+
+func TestBackfillSnapshotRowFromHourlyCacheNoMatch(t *testing.T) {
+	ctx := context.Background()
+	dbConn := newTasksTestDB(t)
+
+	if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil {
+		t.Fatalf("failed to ensure vm_hourly_stats: %v", err)
+	}
+
+	row := InventorySnapshotRow{
+		Vcenter: "vc-a",
+		VmId:    sql.NullString{String: "vm-missing", Valid: true},
+	}
+	changed := backfillSnapshotRowFromHourlyCache(ctx, dbConn, &row)
+	if changed {
+		t.Fatal("expected no backfill change for missing VM")
+	}
+}

internal/vcenter/vcenter.go

@@ -296,6 +296,24 @@ func (v *Vcenter) GetAllVMsWithProps() ([]mo.VirtualMachine, error) {
 	return vms, nil
 }
 
+// GetVMWithSnapshotPropsByRef retrieves snapshot-critical properties for a single VM reference.
+// Used as a targeted fallback when bulk retrieval returns partial data for a VM.
+func (v *Vcenter) GetVMWithSnapshotPropsByRef(ref types.ManagedObjectReference) (*mo.VirtualMachine, error) {
+	var vm mo.VirtualMachine
+	props := []string{
+		"name",
+		"parent",
+		"config",
+		"runtime.powerState",
+		"runtime.host",
+		"resourcePool",
+	}
+	if err := v.client.RetrieveOne(v.ctx, ref, props, &vm); err != nil {
+		return nil, err
+	}
+	return &vm, nil
+}
+
 // FindVmDeletionEvents returns a map of MoRef (VmId) to the deletion event time within the given window.
 func (v *Vcenter) FindVmDeletionEvents(ctx context.Context, begin, end time.Time) (map[string]time.Time, error) {
 	return v.findVmDeletionEvents(ctx, begin, end, nil)