update aggregation jobs

internal/tasks/dailyAggregate.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"time"
 	"vctp/db"
+	"vctp/db/queries"
 	"vctp/internal/metrics"
 	"vctp/internal/report"
 )
@@ -264,6 +265,9 @@ func (c *CronTask) aggregateDailySummaryGo(ctx context.Context, dayStart, dayEnd
 		sort.Slice(snapTimes, func(i, j int) bool { return snapTimes[i] < snapTimes[j] })
 	}
 
+	inventoryDeletions := c.applyInventoryDeletions(ctx, aggMap)
+	c.Logger.Info("Daily aggregation deletion times", "source_inventory", inventoryDeletions)
+
 	// Get the first hourly snapshot on/after dayEnd to help confirm deletions that happen on the last snapshot of the day.
 	var nextSnapshotTable string
 	nextSnapshotRows, nextErr := c.Database.DB().QueryxContext(ctx, `
@@ -308,7 +312,11 @@ LIMIT 1
 		maxSnap = snapTimes[len(snapTimes)-1]
 	}
 
+	inferredDeletions := 0
 	for _, v := range aggMap {
+		if v.deletion != 0 {
+			continue
+		}
 		// Infer deletion only after seeing at least two consecutive absent snapshots after lastSeen.
 		if maxSnap > 0 && len(v.seen) > 0 && v.lastSeen < maxSnap {
 			c.Logger.Debug("inferring deletion window", "vm_id", v.key.VmId, "vm_uuid", v.key.VmUuid, "name", v.key.Name, "last_seen", v.lastSeen, "snapshots", len(snapTimes))
@@ -330,6 +338,7 @@ LIMIT 1
 			}
 			if consecutiveMisses >= 2 {
 				v.deletion = firstMiss
+				inferredDeletions++
 				break
 			}
 		}
@@ -341,6 +350,7 @@ LIMIT 1
 				_, presentByName := nextPresence["name:"+v.key.Name]
 				if !presentByID && !presentByUUID && !presentByName {
 					v.deletion = firstMiss
+					inferredDeletions++
 					c.Logger.Debug("cross-day deletion inferred from next snapshot", "vm_id", v.key.VmId, "vm_uuid", v.key.VmUuid, "name", v.key.Name, "deletion", firstMiss, "next_table", nextSnapshotTable)
 				}
 			}
@@ -349,6 +359,7 @@ LIMIT 1
 			}
 		}
 	}
+	c.Logger.Info("Daily aggregation deletion times", "source_inferred", inferredDeletions)
 
 	// Insert aggregated rows.
 	if err := c.insertDailyAggregates(ctx, summaryTable, aggMap, totalSamples); err != nil {
@@ -392,6 +403,61 @@ LIMIT 1
 	return nil
 }
 
+func (c *CronTask) applyInventoryDeletions(ctx context.Context, agg map[dailyAggKey]*dailyAggVal) int {
+	dbConn := c.Database.DB()
+	vcenters := make(map[string]struct{}, 8)
+	for k := range agg {
+		if k.Vcenter != "" {
+			vcenters[k.Vcenter] = struct{}{}
+		}
+	}
+	totalApplied := 0
+	for vcenter := range vcenters {
+		inventoryRows, err := queries.New(dbConn).GetInventoryByVcenter(ctx, vcenter)
+		if err != nil {
+			c.Logger.Warn("failed to load inventory for daily deletion times", "vcenter", vcenter, "error", err)
+			continue
+		}
+		byID := make(map[string]int64, len(inventoryRows))
+		byUUID := make(map[string]int64, len(inventoryRows))
+		byName := make(map[string]int64, len(inventoryRows))
+		for _, inv := range inventoryRows {
+			if !inv.DeletionTime.Valid || inv.DeletionTime.Int64 <= 0 {
+				continue
+			}
+			if inv.VmId.Valid && strings.TrimSpace(inv.VmId.String) != "" {
+				byID[strings.TrimSpace(inv.VmId.String)] = inv.DeletionTime.Int64
+			}
+			if inv.VmUuid.Valid && strings.TrimSpace(inv.VmUuid.String) != "" {
+				byUUID[strings.TrimSpace(inv.VmUuid.String)] = inv.DeletionTime.Int64
+			}
+			if strings.TrimSpace(inv.Name) != "" {
+				byName[strings.ToLower(strings.TrimSpace(inv.Name))] = inv.DeletionTime.Int64
+			}
+		}
+		for k, v := range agg {
+			if v.deletion != 0 || k.Vcenter != vcenter {
+				continue
+			}
+			if ts, ok := byID[k.VmId]; ok {
+				v.deletion = ts
+				totalApplied++
+				continue
+			}
+			if ts, ok := byUUID[k.VmUuid]; ok {
+				v.deletion = ts
+				totalApplied++
+				continue
+			}
+			if ts, ok := byName[strings.ToLower(k.Name)]; ok {
+				v.deletion = ts
+				totalApplied++
+			}
+		}
+	}
+	return totalApplied
+}
+
 func (c *CronTask) scanHourlyTablesParallel(ctx context.Context, snapshots []report.SnapshotRecord) (map[dailyAggKey]*dailyAggVal, error) {
 	agg := make(map[dailyAggKey]*dailyAggVal, 1024)
 	mu := sync.Mutex{}
@@ -555,6 +621,9 @@ FROM %s
 			goldHits:       hitGold,
 			seen:           map[int64]struct{}{int64OrZero(snapshotTime): {}},
 		}
+		if deletion.Valid && deletion.Int64 > 0 {
+			row.deletion = deletion.Int64
+		}
 		out[key] = row
 	}
 	return out, nil