improve vm deletion detection

internal/vcenter/vcenter.go

@@ -199,6 +199,28 @@ func (v *Vcenter) GetAllVMsWithProps() ([]mo.VirtualMachine, error) {
 
 // FindVmDeletionEvents returns a map of MoRef (VmId) to the deletion event time within the given window.
 func (v *Vcenter) FindVmDeletionEvents(ctx context.Context, begin, end time.Time) (map[string]time.Time, error) {
+	return v.findVmDeletionEvents(ctx, begin, end, nil)
+}
+
+// FindVmDeletionEventsForCandidates returns deletion event times for the provided VM IDs only.
+func (v *Vcenter) FindVmDeletionEventsForCandidates(ctx context.Context, begin, end time.Time, candidates []string) (map[string]time.Time, error) {
+	if len(candidates) == 0 {
+		return map[string]time.Time{}, nil
+	}
+	candidateSet := make(map[string]struct{}, len(candidates))
+	for _, id := range candidates {
+		if id == "" {
+			continue
+		}
+		candidateSet[id] = struct{}{}
+	}
+	if len(candidateSet) == 0 {
+		return map[string]time.Time{}, nil
+	}
+	return v.findVmDeletionEvents(ctx, begin, end, candidateSet)
+}
+
+func (v *Vcenter) findVmDeletionEvents(ctx context.Context, begin, end time.Time, candidateSet map[string]struct{}) (map[string]time.Time, error) {
 	result := make(map[string]time.Time)
 	if v.client == nil || !v.client.Valid() {
 		return result, fmt.Errorf("vcenter client is not valid")
@@ -208,18 +230,63 @@ func (v *Vcenter) FindVmDeletionEvents(ctx context.Context, begin, end time.Time
 	endUTC := end.UTC()
 	mgr := event.NewManager(v.client.Client)
 
-	recordDeletion := func(vmID string, ts time.Time) {
+	type deletionHit struct {
+		ts       time.Time
+		priority int
+	}
+	const (
+		deletionPriorityRemoved = iota
+		deletionPriorityVmEvent
+		deletionPriorityTask
+	)
+	hits := make(map[string]deletionHit)
+	foundCandidates := 0
+	recordDeletion := func(vmID string, ts time.Time, priority int) {
 		if vmID == "" {
 			return
 		}
-		if prev, ok := result[vmID]; !ok || ts.Before(prev) {
-			result[vmID] = ts
+		if candidateSet != nil {
+			if _, ok := candidateSet[vmID]; !ok {
+				return
+			}
+		}
+		if prev, ok := hits[vmID]; !ok {
+			hits[vmID] = deletionHit{ts: ts, priority: priority}
+			if candidateSet != nil {
+				foundCandidates++
+			}
+		} else if priority < prev.priority || (priority == prev.priority && ts.Before(prev.ts)) {
+			hits[vmID] = deletionHit{ts: ts, priority: priority}
 		}
 	}
 
 	isDeletionMessage := func(msg string) bool {
 		msg = strings.ToLower(msg)
-		return strings.Contains(msg, "destroy") || strings.Contains(msg, "deleted")
+		return strings.Contains(msg, "destroy") ||
+			strings.Contains(msg, "deleted") ||
+			strings.Contains(msg, "unregister") ||
+			strings.Contains(msg, "removed from inventory")
+	}
+
+	isVmDeletionTask := func(info types.TaskInfo, msg string) bool {
+		id := strings.ToLower(strings.TrimSpace(info.DescriptionId))
+		if id != "" {
+			if strings.Contains(id, "virtualmachine") &&
+				(strings.Contains(id, "destroy") || strings.Contains(id, "delete") || strings.Contains(id, "unregister")) {
+				return true
+			}
+		}
+		name := strings.ToLower(strings.TrimSpace(info.Name))
+		if name != "" {
+			if (strings.Contains(name, "destroy") || strings.Contains(name, "delete") || strings.Contains(name, "unregister")) &&
+				(strings.Contains(name, "virtualmachine") || strings.Contains(name, "virtual machine")) {
+				return true
+			}
+		}
+		if msg != "" && isDeletionMessage(msg) {
+			return true
+		}
+		return false
 	}
 
 	processEvents := func(evts []types.BaseEvent) {
@@ -228,33 +295,67 @@ func (v *Vcenter) FindVmDeletionEvents(ctx context.Context, begin, end time.Time
 			case *types.VmRemovedEvent:
 				if e.Vm != nil {
 					vmID := e.Vm.Vm.Value
-					recordDeletion(vmID, e.CreatedTime)
+					recordDeletion(vmID, e.CreatedTime, deletionPriorityRemoved)
 				}
 			case *types.TaskEvent:
 				// Fallback for destroy task events.
 				if e.Info.Entity != nil {
 					vmID := e.Info.Entity.Value
-					if vmID != "" && isDeletionMessage(e.GetEvent().FullFormattedMessage) {
-						recordDeletion(vmID, e.CreatedTime)
+					if vmID != "" && isVmDeletionTask(e.Info, e.GetEvent().FullFormattedMessage) {
+						recordDeletion(vmID, e.CreatedTime, deletionPriorityTask)
 					}
 				}
 			case *types.VmEvent:
 				if e.Vm != nil {
 					vmID := e.Vm.Vm.Value
 					if vmID != "" && isDeletionMessage(e.GetEvent().FullFormattedMessage) {
-						recordDeletion(vmID, e.CreatedTime)
+						recordDeletion(vmID, e.CreatedTime, deletionPriorityVmEvent)
 					}
 				}
 			}
 		}
 	}
 
+	const (
+		eventPageSize = int32(1000)
+		maxEventPages = 25
+	)
+	readCollector := func(label string, collector *event.HistoryCollector) error {
+		pageCount := 0
+		for {
+			events, err := collector.ReadNextEvents(ctx, eventPageSize)
+			if err != nil {
+				return err
+			}
+			if len(events) == 0 {
+				break
+			}
+			processEvents(events)
+			if candidateSet != nil && foundCandidates >= len(candidateSet) {
+				break
+			}
+			pageCount++
+			if pageCount >= maxEventPages {
+				if v.Logger != nil {
+					v.Logger.Warn("vcenter deletion events truncated", "vcenter", v.Vurl, "label", label, "pages", pageCount, "page_size", eventPageSize, "window_start_utc", beginUTC, "window_end_utc", endUTC)
+				}
+				break
+			}
+			if len(events) < int(eventPageSize) {
+				break
+			}
+		}
+		return nil
+	}
+
 	// First attempt: specific deletion event types.
+	disableFullMessage := false
 	filter := types.EventFilterSpec{
 		Time: &types.EventFilterSpecByTime{
 			BeginTime: &beginUTC,
 			EndTime:   &endUTC,
 		},
+		DisableFullMessage: &disableFullMessage,
 		EventTypeId: []string{
 			"VmRemovedEvent",
 			"TaskEvent",
@@ -266,29 +367,32 @@ func (v *Vcenter) FindVmDeletionEvents(ctx context.Context, begin, end time.Time
 	}
 	defer collector.Destroy(ctx)
 
-	events, err := collector.ReadNextEvents(ctx, 500)
-	if err != nil {
+	if err := readCollector("primary", collector); err != nil {
 		return result, fmt.Errorf("failed to read events: %w", err)
 	}
-	processEvents(events)
 
 	// If nothing found, widen the filter to all event types in the window as a fallback.
-	if len(result) == 0 {
+	if len(hits) == 0 {
 		fallbackFilter := types.EventFilterSpec{
 			Time: &types.EventFilterSpecByTime{
 				BeginTime: &beginUTC,
 				EndTime:   &endUTC,
 			},
+			DisableFullMessage: &disableFullMessage,
 		}
 		fc, err := mgr.CreateCollectorForEvents(ctx, fallbackFilter)
 		if err == nil {
 			defer fc.Destroy(ctx)
-			if evs, readErr := fc.ReadNextEvents(ctx, 500); readErr == nil {
-				processEvents(evs)
+			if readErr := readCollector("fallback", fc); readErr != nil && v.Logger != nil {
+				v.Logger.Warn("vcenter fallback event read failed", "vcenter", v.Vurl, "error", readErr)
 			}
 		}
 	}
 
+	for vmID, hit := range hits {
+		result[vmID] = hit.ts
+	}
+
 	return result, nil
 }