improve vm deletion detection

2026-01-27 14:20:30 +11:00
parent 6981bd9994
commit 38480e52c0
2 changed files with 211 additions and 54 deletions
--- a/internal/tasks/inventorySnapshots.go
+++ b/internal/tasks/inventorySnapshots.go
@@ -1062,52 +1062,105 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
 		if freq <= 0 {
 			freq = time.Hour
 		}
-		begin := startTime.Add(-4 * freq)
-		end := startTime
-		events, err := vc.FindVmDeletionEvents(ctx, begin, end)
-		if err != nil {
-			log.Warn("failed to fetch vcenter deletion events", "vcenter", url, "error", err)
-		} else {
-			log.Debug("fetched vcenter deletion events", "vcenter", url, "count", len(events), "window_start_local", begin, "window_end_local", end, "window_minutes", end.Sub(begin).Minutes(), "window_start_utc", begin.UTC(), "window_end_utc", end.UTC())
-			for _, cand := range candidates {
-				if t, ok := events[cand.vmID]; ok {
-					delTs := sql.NullInt64{Int64: t.Unix(), Valid: true}
-					if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{
-						DeletionTime:   delTs,
-						VmId:           sql.NullString{String: cand.vmID, Valid: cand.vmID != ""},
-						DatacenterName: cand.datacenter,
-					}); err != nil {
-						log.Warn("failed to update inventory deletion time from event", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "vcenter", url, "error", err)
+		candidateIDs := make([]string, 0, len(candidates))
+		candidateSet := make(map[string]struct{}, len(candidates))
+		for _, cand := range candidates {
+			if cand.vmID == "" {
+				continue
+			}
+			if _, ok := candidateSet[cand.vmID]; ok {
+				continue
+			}
+			candidateSet[cand.vmID] = struct{}{}
+			candidateIDs = append(candidateIDs, cand.vmID)
+		}
+		events := make(map[string]time.Time)
+		var windowBegin time.Time
+		var windowEnd time.Time
+		var windowUsed time.Duration
+		if len(candidateIDs) > 0 {
+			baseWindow := 4 * freq
+			maxWindow := 24 * time.Hour
+			windowSizes := make([]time.Duration, 0, 3)
+			addWindow := func(d time.Duration) {
+				if d <= 0 {
+					return
+				}
+				if d > maxWindow {
+					d = maxWindow
+				}
+				if len(windowSizes) == 0 || windowSizes[len(windowSizes)-1] != d {
+					windowSizes = append(windowSizes, d)
+				}
+			}
+			addWindow(baseWindow)
+			addWindow(baseWindow * 3)
+			addWindow(baseWindow * 6)
+			for idx, window := range windowSizes {
+				begin := startTime.Add(-window)
+				end := startTime
+				windowEvents, err := vc.FindVmDeletionEventsForCandidates(ctx, begin, end, candidateIDs)
+				if err != nil {
+					log.Warn("failed to fetch vcenter deletion events", "vcenter", url, "error", err, "window_start_local", begin, "window_end_local", end)
+					continue
+				}
+				windowBegin = begin
+				windowEnd = end
+				windowUsed = window
+				for vmID, ts := range windowEvents {
+					if prev, ok := events[vmID]; !ok || ts.Before(prev) {
+						events[vmID] = ts
 					}
-					if err := db.MarkVmDeletedWithDetails(ctx, dbConn, url, cand.vmID, cand.vmUUID, cand.name, cand.cluster, t.Unix()); err != nil {
-						log.Warn("failed to refine lifecycle cache deletion time", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "vcenter", url, "error", err)
+				}
+				if len(events) < len(candidateIDs) && idx < len(windowSizes)-1 {
+					log.Debug("widening deletion event window", "vcenter", url, "matched", len(events), "candidates", len(candidateIDs), "window_minutes", window.Minutes())
+				}
+				if len(events) >= len(candidateIDs) {
+					break
+				}
+			}
+		}
+		if len(events) > 0 {
+			log.Debug("fetched vcenter deletion events", "vcenter", url, "count", len(events), "window_start_local", windowBegin, "window_end_local", windowEnd, "window_minutes", windowUsed.Minutes(), "window_start_utc", windowBegin.UTC(), "window_end_utc", windowEnd.UTC())
+		}
+		for _, cand := range candidates {
+			if t, ok := events[cand.vmID]; ok {
+				delTs := sql.NullInt64{Int64: t.Unix(), Valid: true}
+				if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{
+					DeletionTime:   delTs,
+					VmId:           sql.NullString{String: cand.vmID, Valid: cand.vmID != ""},
+					DatacenterName: cand.datacenter,
+				}); err != nil {
+					log.Warn("failed to update inventory deletion time from event", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "vcenter", url, "error", err)
+				}
+				if err := db.MarkVmDeletedWithDetails(ctx, dbConn, url, cand.vmID, cand.vmUUID, cand.name, cand.cluster, t.Unix()); err != nil {
+					log.Warn("failed to refine lifecycle cache deletion time", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "vcenter", url, "error", err)
+				}
+				if snapRow, snapTable, found := findVMInHourlySnapshots(ctx, dbConn, url, cand.vmID); found {
+					vmUUID := cand.vmUUID
+					if vmUUID == "" && snapRow.VmUuid.Valid {
+						vmUUID = snapRow.VmUuid.String
 					}
-					if snapRow, snapTable, found := findVMInHourlySnapshots(ctx, dbConn, url, cand.vmID); found {
-						vmUUID := cand.vmUUID
-						if vmUUID == "" && snapRow.VmUuid.Valid {
-							vmUUID = snapRow.VmUuid.String
-						}
-						name := cand.name
-						if name == "" {
-							name = snapRow.Name
-						}
-						if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, snapTable, url, cand.vmID, vmUUID, name, delTs.Int64); err != nil {
-							log.Warn("failed to update hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "error", err)
-						} else if rowsAffected > 0 {
-							reportTables[snapTable] = struct{}{}
-							deletionsMarked = true
-							log.Debug("updated hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "event_time", t)
-							if snapUnix, ok := parseSnapshotTime(snapTable); ok {
-								if cacheRows, err := updateDeletionTimeInHourlyCache(ctx, dbConn, url, cand.vmID, vmUUID, name, snapUnix, delTs.Int64); err != nil {
-									log.Warn("failed to update hourly cache deletion time from event", "snapshot_time", snapUnix, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "error", err)
-								} else if cacheRows > 0 {
-									log.Debug("updated hourly cache deletion time from event", "snapshot_time", snapUnix, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "event_time", t)
-								}
+					name := cand.name
+					if name == "" {
+						name = snapRow.Name
+					}
+					if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, snapTable, url, cand.vmID, vmUUID, name, delTs.Int64); err != nil {
+						log.Warn("failed to update hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "error", err)
+					} else if rowsAffected > 0 {
+						reportTables[snapTable] = struct{}{}
+						deletionsMarked = true
+						log.Debug("updated hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "event_time", t)
+						if snapUnix, ok := parseSnapshotTime(snapTable); ok {
+							if cacheRows, err := updateDeletionTimeInHourlyCache(ctx, dbConn, url, cand.vmID, vmUUID, name, snapUnix, delTs.Int64); err != nil {
+								log.Warn("failed to update hourly cache deletion time from event", "snapshot_time", snapUnix, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "error", err)
+							} else if cacheRows > 0 {
+								log.Debug("updated hourly cache deletion time from event", "snapshot_time", snapUnix, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "event_time", t)
 							}
 						}
 					}
-					log.Info("refined deletion time from vcenter event", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "vcenter", url, "event_time", t)
 				}
+				log.Info("refined deletion time from vcenter event", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "vcenter", url, "event_time", t)
 			}
 		}
 	}