more accurate resource pool data in aggregation reports
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
@@ -179,13 +179,46 @@ func querySnapshotRows(ctx context.Context, dbConn *sqlx.DB, table string, colum
|
||||
return dbConn.QueryxContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func updateDeletionTimeInSnapshot(ctx context.Context, dbConn *sqlx.DB, table, vcenter, vmID, vmUUID, name string, deletionUnix int64) (int64, error) {
|
||||
if err := db.ValidateTableName(table); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
matchColumn := ""
|
||||
matchValue := ""
|
||||
switch {
|
||||
case vmID != "":
|
||||
matchColumn = "VmId"
|
||||
matchValue = vmID
|
||||
case vmUUID != "":
|
||||
matchColumn = "VmUuid"
|
||||
matchValue = vmUUID
|
||||
case name != "":
|
||||
matchColumn = "Name"
|
||||
matchValue = name
|
||||
default:
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(`UPDATE %s SET "DeletionTime" = ? WHERE "Vcenter" = ? AND "%s" = ? AND ("DeletionTime" IS NULL OR "DeletionTime" = 0 OR "DeletionTime" > ?)`, table, matchColumn)
|
||||
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
|
||||
result, err := dbConn.ExecContext(ctx, query, deletionUnix, vcenter, matchValue, deletionUnix)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return rowsAffected, nil
|
||||
}
|
||||
|
||||
// markMissingFromPrevious marks VMs that were present in the previous snapshot but missing now.
|
||||
func (c *CronTask) markMissingFromPrevious(ctx context.Context, dbConn *sqlx.DB, prevTable string, vcenter string, snapshotTime time.Time,
|
||||
currentByID map[string]InventorySnapshotRow, currentByUuid map[string]struct{}, currentByName map[string]struct{},
|
||||
invByID map[string]queries.Inventory, invByUuid map[string]queries.Inventory, invByName map[string]queries.Inventory) int {
|
||||
invByID map[string]queries.Inventory, invByUuid map[string]queries.Inventory, invByName map[string]queries.Inventory) (int, bool) {
|
||||
|
||||
if err := db.ValidateTableName(prevTable); err != nil {
|
||||
return 0
|
||||
return 0, false
|
||||
}
|
||||
|
||||
type prevRow struct {
|
||||
@@ -200,11 +233,12 @@ func (c *CronTask) markMissingFromPrevious(ctx context.Context, dbConn *sqlx.DB,
|
||||
rows, err := querySnapshotRows(ctx, dbConn, prevTable, []string{"VmId", "VmUuid", "Name", "Cluster", "Datacenter", "DeletionTime"}, `"Vcenter" = ?`, vcenter)
|
||||
if err != nil {
|
||||
c.Logger.Warn("failed to read previous snapshot for deletion detection", "error", err, "table", prevTable, "vcenter", vcenter)
|
||||
return 0
|
||||
return 0, false
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
missing := 0
|
||||
tableUpdated := false
|
||||
for rows.Next() {
|
||||
var r prevRow
|
||||
if err := rows.StructScan(&r); err != nil {
|
||||
@@ -255,18 +289,16 @@ func (c *CronTask) markMissingFromPrevious(ctx context.Context, dbConn *sqlx.DB,
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if inv.DeletionTime.Valid {
|
||||
continue
|
||||
}
|
||||
|
||||
delTime := sql.NullInt64{Int64: snapshotTime.Unix(), Valid: true}
|
||||
if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{
|
||||
DeletionTime: delTime,
|
||||
VmId: inv.VmId,
|
||||
DatacenterName: inv.Datacenter,
|
||||
}); err != nil {
|
||||
c.Logger.Warn("failed to mark inventory record deleted from previous snapshot", "error", err, "vm_id", inv.VmId.String)
|
||||
continue
|
||||
delTime := inv.DeletionTime
|
||||
if !delTime.Valid {
|
||||
delTime = sql.NullInt64{Int64: snapshotTime.Unix(), Valid: true}
|
||||
if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{
|
||||
DeletionTime: delTime,
|
||||
VmId: inv.VmId,
|
||||
DatacenterName: inv.Datacenter,
|
||||
}); err != nil {
|
||||
c.Logger.Warn("failed to mark inventory record deleted from previous snapshot", "error", err, "vm_id", inv.VmId.String)
|
||||
}
|
||||
}
|
||||
// Also update lifecycle cache so deletion time is available for rollups.
|
||||
vmUUID := ""
|
||||
@@ -276,11 +308,17 @@ func (c *CronTask) markMissingFromPrevious(ctx context.Context, dbConn *sqlx.DB,
|
||||
if err := db.MarkVmDeletedWithDetails(ctx, dbConn, vcenter, inv.VmId.String, vmUUID, inv.Name, inv.Cluster.String, delTime.Int64); err != nil {
|
||||
c.Logger.Warn("failed to mark lifecycle cache deleted from previous snapshot", "error", err, "vm_id", inv.VmId.String, "vm_uuid", vmUUID, "vcenter", vcenter)
|
||||
}
|
||||
if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, prevTable, vcenter, inv.VmId.String, vmUUID, inv.Name, delTime.Int64); err != nil {
|
||||
c.Logger.Warn("failed to update hourly snapshot deletion time", "error", err, "table", prevTable, "vm_id", inv.VmId.String, "vm_uuid", vmUUID, "vcenter", vcenter)
|
||||
} else if rowsAffected > 0 {
|
||||
tableUpdated = true
|
||||
c.Logger.Debug("updated hourly snapshot deletion time", "table", prevTable, "vm_id", inv.VmId.String, "vm_uuid", vmUUID, "vcenter", vcenter, "deletion_time", delTime.Int64)
|
||||
}
|
||||
c.Logger.Debug("Detected VM missing compared to previous snapshot", "name", inv.Name, "vm_id", inv.VmId.String, "vm_uuid", inv.VmUuid.String, "vcenter", vcenter, "snapshot_time", snapshotTime, "prev_table", prevTable)
|
||||
missing++
|
||||
}
|
||||
|
||||
return missing
|
||||
return missing, tableUpdated
|
||||
}
|
||||
|
||||
// countNewFromPrevious returns how many VMs are present in the current snapshot but not in the previous snapshot.
|
||||
@@ -409,9 +447,9 @@ func listNewFromPrevious(ctx context.Context, dbConn *sqlx.DB, prevTable string,
|
||||
|
||||
// findVMInHourlySnapshots searches recent hourly snapshot tables for a VM by ID for the given vCenter.
|
||||
// extraTables are searched first (e.g., known previous snapshot tables).
|
||||
func findVMInHourlySnapshots(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID string, extraTables ...string) (InventorySnapshotRow, bool) {
|
||||
func findVMInHourlySnapshots(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID string, extraTables ...string) (InventorySnapshotRow, string, bool) {
|
||||
if vmID == "" {
|
||||
return InventorySnapshotRow{}, false
|
||||
return InventorySnapshotRow{}, "", false
|
||||
}
|
||||
// Use a short timeout to avoid hanging if the DB is busy.
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
@@ -429,7 +467,7 @@ func findVMInHourlySnapshots(ctx context.Context, dbConn *sqlx.DB, vcenter strin
|
||||
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
|
||||
var row InventorySnapshotRow
|
||||
if err := dbConn.QueryRowxContext(ctx, query, vcenter, vmID).Scan(&row.VmId, &row.VmUuid, &row.Name, &row.Datacenter, &row.Cluster); err == nil {
|
||||
return row, true
|
||||
return row, table, true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -442,7 +480,7 @@ ORDER BY snapshot_time DESC
|
||||
LIMIT 20
|
||||
`)
|
||||
if err != nil {
|
||||
return InventorySnapshotRow{}, false
|
||||
return InventorySnapshotRow{}, "", false
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
@@ -459,12 +497,12 @@ LIMIT 20
|
||||
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
|
||||
var row InventorySnapshotRow
|
||||
if err := dbConn.QueryRowxContext(ctx, query, vcenter, vmID).Scan(&row.VmId, &row.VmUuid, &row.Name, &row.Datacenter, &row.Cluster); err == nil {
|
||||
return row, true
|
||||
return row, table, true
|
||||
}
|
||||
checked++
|
||||
if checked >= 10 { // limit work
|
||||
break
|
||||
}
|
||||
}
|
||||
return InventorySnapshotRow{}, false
|
||||
return InventorySnapshotRow{}, "", false
|
||||
}
|
||||
|
||||
@@ -28,21 +28,22 @@ func presenceKeys(vmID, vmUUID, name string) []string {
|
||||
|
||||
// backfillLifecycleDeletionsToday looks for VMs in the lifecycle cache that are not in the current inventory,
|
||||
// have no DeletedAt, and determines their deletion time from today's hourly snapshots, optionally checking the next snapshot (next day) to confirm.
|
||||
func backfillLifecycleDeletionsToday(ctx context.Context, logger *slog.Logger, dbConn *sqlx.DB, vcenter string, snapshotTime time.Time, present map[string]InventorySnapshotRow) error {
|
||||
// It returns any hourly snapshot tables that were updated with deletion times.
|
||||
func backfillLifecycleDeletionsToday(ctx context.Context, logger *slog.Logger, dbConn *sqlx.DB, vcenter string, snapshotTime time.Time, present map[string]InventorySnapshotRow) ([]string, error) {
|
||||
dayStart := truncateDate(snapshotTime)
|
||||
dayEnd := dayStart.Add(24 * time.Hour)
|
||||
|
||||
candidates, err := loadLifecycleCandidates(ctx, dbConn, vcenter, present)
|
||||
if err != nil || len(candidates) == 0 {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tables, err := listHourlyTablesForDay(ctx, dbConn, dayStart, dayEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if len(tables) == 0 {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
nextPresence := make(map[string]struct{})
|
||||
@@ -50,10 +51,12 @@ func backfillLifecycleDeletionsToday(ctx context.Context, logger *slog.Logger, d
|
||||
nextPresence = loadPresenceKeys(ctx, dbConn, nextTable, vcenter)
|
||||
}
|
||||
|
||||
for _, cand := range candidates {
|
||||
deletion, firstMiss := findDeletionInTables(ctx, dbConn, tables, vcenter, cand)
|
||||
updatedTables := make(map[string]struct{})
|
||||
for i := range candidates {
|
||||
cand := &candidates[i]
|
||||
deletion, firstMiss, lastSeenTable := findDeletionInTables(ctx, dbConn, tables, vcenter, cand)
|
||||
if deletion == 0 && len(nextPresence) > 0 && firstMiss > 0 {
|
||||
if !isPresent(nextPresence, cand) {
|
||||
if !isPresent(nextPresence, *cand) {
|
||||
// Single miss at end of day, confirmed by next-day absence.
|
||||
deletion = firstMiss
|
||||
logger.Debug("cross-day deletion inferred from next snapshot", "vcenter", vcenter, "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "deletion", deletion)
|
||||
@@ -64,10 +67,25 @@ func backfillLifecycleDeletionsToday(ctx context.Context, logger *slog.Logger, d
|
||||
logger.Warn("lifecycle backfill mark deleted failed", "vcenter", vcenter, "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "cluster", cand.cluster, "deletion", deletion, "error", err)
|
||||
continue
|
||||
}
|
||||
if lastSeenTable != "" {
|
||||
if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, lastSeenTable, vcenter, cand.vmID, cand.vmUUID, cand.name, deletion); err != nil {
|
||||
logger.Warn("lifecycle backfill failed to update hourly snapshot deletion time", "vcenter", vcenter, "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "cluster", cand.cluster, "table", lastSeenTable, "deletion", deletion, "error", err)
|
||||
} else if rowsAffected > 0 {
|
||||
updatedTables[lastSeenTable] = struct{}{}
|
||||
logger.Debug("lifecycle backfill updated hourly snapshot deletion time", "vcenter", vcenter, "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "cluster", cand.cluster, "table", lastSeenTable, "deletion", deletion)
|
||||
}
|
||||
}
|
||||
logger.Debug("lifecycle backfill applied", "vcenter", vcenter, "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "cluster", cand.cluster, "deletion", deletion)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if len(updatedTables) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
tablesUpdated := make([]string, 0, len(updatedTables))
|
||||
for table := range updatedTables {
|
||||
tablesUpdated = append(tablesUpdated, table)
|
||||
}
|
||||
return tablesUpdated, nil
|
||||
}
|
||||
|
||||
type lifecycleCandidate struct {
|
||||
@@ -212,41 +230,47 @@ func isPresent(presence map[string]struct{}, cand lifecycleCandidate) bool {
|
||||
}
|
||||
|
||||
// findDeletionInTables walks ordered hourly tables for a vCenter and returns the first confirmed deletion time
|
||||
// (requiring two consecutive misses) plus the time of the first miss for cross-day handling.
|
||||
func findDeletionInTables(ctx context.Context, dbConn *sqlx.DB, tables []snapshotTable, vcenter string, cand lifecycleCandidate) (int64, int64) {
|
||||
// (requiring two consecutive misses), the time of the first miss for cross-day handling, and the last table where
|
||||
// the VM was seen so we can backfill deletion time into that snapshot.
|
||||
func findDeletionInTables(ctx context.Context, dbConn *sqlx.DB, tables []snapshotTable, vcenter string, cand *lifecycleCandidate) (int64, int64, string) {
|
||||
var lastSeen int64
|
||||
var lastSeenTable string
|
||||
var firstMiss int64
|
||||
for i, tbl := range tables {
|
||||
rows, err := querySnapshotRows(ctx, dbConn, tbl.Table, []string{"VmId", "VmUuid", "Name", "Cluster"}, `"Vcenter" = ? AND "VmId" = ?`, vcenter, cand.vmID)
|
||||
if err == nil {
|
||||
if rows.Next() {
|
||||
var vmId, vmUuid, name, cluster sql.NullString
|
||||
if scanErr := rows.Scan(&vmId, &vmUuid, &name, &cluster); scanErr == nil {
|
||||
lastSeen = tbl.Time
|
||||
if cand.name == "" && name.Valid {
|
||||
cand.name = name.String
|
||||
}
|
||||
if cand.cluster == "" && cluster.Valid {
|
||||
cand.cluster = cluster.String
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
seen := false
|
||||
if rows.Next() {
|
||||
var vmId, vmUuid, name, cluster sql.NullString
|
||||
if scanErr := rows.Scan(&vmId, &vmUuid, &name, &cluster); scanErr == nil {
|
||||
seen = true
|
||||
lastSeen = tbl.Time
|
||||
lastSeenTable = tbl.Table
|
||||
if cand.vmUUID == "" && vmUuid.Valid {
|
||||
cand.vmUUID = vmUuid.String
|
||||
}
|
||||
if cand.name == "" && name.Valid {
|
||||
cand.name = name.String
|
||||
}
|
||||
if cand.cluster == "" && cluster.Valid {
|
||||
cand.cluster = cluster.String
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
if lastSeen > 0 && tbl.Time > lastSeen {
|
||||
// first table after last seen -> first miss
|
||||
if seen, _ := candSeenInTable(ctx, dbConn, tbl.Table, vcenter, cand.vmID); !seen {
|
||||
firstMiss = tbl.Time
|
||||
// need two consecutive misses
|
||||
if i+1 < len(tables) {
|
||||
if seen2, _ := candSeenInTable(ctx, dbConn, tables[i+1].Table, vcenter, cand.vmID); !seen2 {
|
||||
return firstMiss, firstMiss
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
if lastSeen > 0 && !seen && firstMiss == 0 {
|
||||
firstMiss = tbl.Time
|
||||
if i+1 < len(tables) {
|
||||
if seen2, _ := candSeenInTable(ctx, dbConn, tables[i+1].Table, vcenter, cand.vmID); !seen2 {
|
||||
return firstMiss, firstMiss, lastSeenTable
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, firstMiss
|
||||
return 0, firstMiss, lastSeenTable
|
||||
}
|
||||
|
||||
func candSeenInTable(ctx context.Context, dbConn *sqlx.DB, table, vcenter, vmID string) (bool, error) {
|
||||
|
||||
@@ -1057,6 +1057,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
missingCount, deletionsMarked, candidates := prepareDeletionCandidates(ctx, log, dbConn, url, inventoryRows, presentSnapshots, presentByUuid, presentByName, startTime)
|
||||
newCount := 0
|
||||
prevTableName := ""
|
||||
reportTables := make(map[string]struct{})
|
||||
|
||||
// If deletions detected, refine deletion time using vCenter events in a small window.
|
||||
if missingCount > 0 {
|
||||
@@ -1084,6 +1085,23 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
if err := db.MarkVmDeletedWithDetails(ctx, dbConn, url, cand.vmID, cand.vmUUID, cand.name, cand.cluster, t.Unix()); err != nil {
|
||||
log.Warn("failed to refine lifecycle cache deletion time", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "vcenter", url, "error", err)
|
||||
}
|
||||
if snapRow, snapTable, found := findVMInHourlySnapshots(ctx, dbConn, url, cand.vmID); found {
|
||||
vmUUID := cand.vmUUID
|
||||
if vmUUID == "" && snapRow.VmUuid.Valid {
|
||||
vmUUID = snapRow.VmUuid.String
|
||||
}
|
||||
name := cand.name
|
||||
if name == "" {
|
||||
name = snapRow.Name
|
||||
}
|
||||
if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, snapTable, url, cand.vmID, vmUUID, name, delTs.Int64); err != nil {
|
||||
log.Warn("failed to update hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "error", err)
|
||||
} else if rowsAffected > 0 {
|
||||
reportTables[snapTable] = struct{}{}
|
||||
deletionsMarked = true
|
||||
log.Debug("updated hourly snapshot deletion time from event", "table", snapTable, "vm_id", cand.vmID, "vm_uuid", vmUUID, "vcenter", url, "event_time", t)
|
||||
}
|
||||
}
|
||||
log.Info("refined deletion time from vcenter event", "vm_id", cand.vmID, "vm_uuid", cand.vmUUID, "name", cand.name, "vcenter", url, "event_time", t)
|
||||
}
|
||||
}
|
||||
@@ -1124,17 +1142,26 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
}
|
||||
|
||||
// Discover previous snapshots once per run (serial) to avoid concurrent probes across vCenters.
|
||||
prevTableName, newCount, missingCount = c.compareWithPreviousSnapshot(ctx, dbConn, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName, missingCount)
|
||||
var prevTableTouched bool
|
||||
prevTableName, newCount, missingCount, prevTableTouched = c.compareWithPreviousSnapshot(ctx, dbConn, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName, missingCount)
|
||||
if prevTableTouched && prevTableName != "" {
|
||||
reportTables[prevTableName] = struct{}{}
|
||||
deletionsMarked = true
|
||||
}
|
||||
|
||||
// If VM count dropped versus totals and we still haven't marked missing, try another comparison + wider event window.
|
||||
if missingCount == 0 && prevVmCount.Valid && prevVmCount.Int64 > int64(totals.VmCount) {
|
||||
// Fallback: locate a previous table only if we didn't already find one.
|
||||
if prevTableName == "" {
|
||||
if prevTable, err := latestHourlySnapshotBefore(ctx, dbConn, startTime, loggerFromCtx(ctx, c.Logger)); err == nil && prevTable != "" {
|
||||
moreMissing := c.markMissingFromPrevious(ctx, dbConn, prevTable, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
|
||||
moreMissing, tableUpdated := c.markMissingFromPrevious(ctx, dbConn, prevTable, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
|
||||
if moreMissing > 0 {
|
||||
missingCount += moreMissing
|
||||
}
|
||||
if tableUpdated {
|
||||
reportTables[prevTable] = struct{}{}
|
||||
deletionsMarked = true
|
||||
}
|
||||
// Reuse this table name for later snapshot lookups when correlating deletion events.
|
||||
prevTableName = prevTable
|
||||
}
|
||||
@@ -1157,9 +1184,10 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
}
|
||||
inv, ok := inventoryByVmID[vmID]
|
||||
var snapRow InventorySnapshotRow
|
||||
var snapTable string
|
||||
if !ok {
|
||||
var found bool
|
||||
snapRow, found = findVMInHourlySnapshots(ctx, dbConn, url, vmID, prevTableName)
|
||||
snapRow, snapTable, found = findVMInHourlySnapshots(ctx, dbConn, url, vmID, prevTableName)
|
||||
if !found {
|
||||
c.Logger.Debug("count-drop: deletion event has no snapshot match", "vm_id", vmID, "vcenter", url, "event_time", t)
|
||||
continue
|
||||
@@ -1170,7 +1198,7 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
Name: snapRow.Name,
|
||||
Datacenter: snapRow.Datacenter,
|
||||
}
|
||||
c.Logger.Info("count-drop: correlated deletion via snapshot lookup", "vm_id", vmID, "vm_uuid", inv.VmUuid.String, "name", inv.Name, "vcenter", url, "event_time", t, "snapshot_table", prevTableName)
|
||||
c.Logger.Info("count-drop: correlated deletion via snapshot lookup", "vm_id", vmID, "vm_uuid", inv.VmUuid.String, "name", inv.Name, "vcenter", url, "event_time", t, "snapshot_table", snapTable)
|
||||
}
|
||||
// Prefer UUID from snapshot if inventory entry lacks it.
|
||||
if !inv.VmUuid.Valid && snapRow.VmUuid.Valid {
|
||||
@@ -1193,6 +1221,19 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
if err := db.MarkVmDeletedWithDetails(ctx, dbConn, url, vmID, inv.VmUuid.String, inv.Name, clusterName, t.Unix()); err != nil {
|
||||
c.Logger.Warn("count-drop: failed to refine lifecycle cache deletion time", "vm_id", vmID, "vm_uuid", inv.VmUuid, "vcenter", url, "error", err)
|
||||
}
|
||||
tableToUpdate := snapTable
|
||||
if tableToUpdate == "" {
|
||||
tableToUpdate = prevTableName
|
||||
}
|
||||
if tableToUpdate != "" {
|
||||
if rowsAffected, err := updateDeletionTimeInSnapshot(ctx, dbConn, tableToUpdate, url, vmID, inv.VmUuid.String, inv.Name, delTs.Int64); err != nil {
|
||||
c.Logger.Warn("count-drop: failed to update hourly snapshot deletion time from event", "table", tableToUpdate, "vm_id", vmID, "vcenter", url, "error", err)
|
||||
} else if rowsAffected > 0 {
|
||||
reportTables[tableToUpdate] = struct{}{}
|
||||
deletionsMarked = true
|
||||
c.Logger.Debug("count-drop: updated hourly snapshot deletion time from event", "table", tableToUpdate, "vm_id", vmID, "vm_uuid", inv.VmUuid.String, "vcenter", url, "event_time", t)
|
||||
}
|
||||
}
|
||||
missingCount++
|
||||
deletionsMarked = true
|
||||
}
|
||||
@@ -1200,8 +1241,13 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
}
|
||||
|
||||
// Backfill lifecycle deletions for VMs missing from inventory and without DeletedAt.
|
||||
if err := backfillLifecycleDeletionsToday(ctx, log, dbConn, url, startTime, presentSnapshots); err != nil {
|
||||
if backfillTables, err := backfillLifecycleDeletionsToday(ctx, log, dbConn, url, startTime, presentSnapshots); err != nil {
|
||||
log.Warn("failed to backfill lifecycle deletions for today", "vcenter", url, "error", err)
|
||||
} else if len(backfillTables) > 0 {
|
||||
for _, table := range backfillTables {
|
||||
reportTables[table] = struct{}{}
|
||||
}
|
||||
deletionsMarked = true
|
||||
}
|
||||
|
||||
log.Info("Hourly snapshot summary",
|
||||
@@ -1219,10 +1265,15 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
log.Warn("failed to record snapshot run", "url", url, "error", upErr)
|
||||
}
|
||||
if deletionsMarked {
|
||||
if err := c.generateReport(ctx, tableName); err != nil {
|
||||
log.Warn("failed to regenerate hourly report after deletions", "error", err, "table", tableName)
|
||||
} else {
|
||||
log.Debug("Regenerated hourly report after deletions", "table", tableName)
|
||||
if len(reportTables) == 0 {
|
||||
reportTables[tableName] = struct{}{}
|
||||
}
|
||||
for reportTable := range reportTables {
|
||||
if err := c.generateReport(ctx, reportTable); err != nil {
|
||||
log.Warn("failed to regenerate hourly report after deletions", "error", err, "table", reportTable)
|
||||
} else {
|
||||
log.Debug("Regenerated hourly report after deletions", "table", reportTable)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -1242,7 +1293,7 @@ func (c *CronTask) compareWithPreviousSnapshot(
|
||||
inventoryByUuid map[string]queries.Inventory,
|
||||
inventoryByName map[string]queries.Inventory,
|
||||
missingCount int,
|
||||
) (string, int, int) {
|
||||
) (string, int, int, bool) {
|
||||
prevTableName, prevTableErr := latestHourlySnapshotBefore(ctx, dbConn, startTime, loggerFromCtx(ctx, c.Logger))
|
||||
if prevTableErr != nil {
|
||||
c.Logger.Warn("failed to locate previous hourly snapshot for deletion comparison", "error", prevTableErr, "url", url)
|
||||
@@ -1251,9 +1302,13 @@ func (c *CronTask) compareWithPreviousSnapshot(
|
||||
prevSnapshotTime, _ := parseSnapshotTime(prevTableName)
|
||||
|
||||
newCount := 0
|
||||
prevTableTouched := false
|
||||
if prevTableName != "" {
|
||||
moreMissing := c.markMissingFromPrevious(ctx, dbConn, prevTableName, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
|
||||
moreMissing, tableUpdated := c.markMissingFromPrevious(ctx, dbConn, prevTableName, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
|
||||
missingCount += moreMissing
|
||||
if tableUpdated {
|
||||
prevTableTouched = true
|
||||
}
|
||||
expectedSeconds := int64(c.Settings.Values.Settings.VcenterInventorySnapshotSeconds) / 2
|
||||
// Skip only if snapshots are closer together than half the configured cadence
|
||||
if SnapshotTooSoon(prevSnapshotTime, startTime.Unix(), expectedSeconds) {
|
||||
@@ -1278,5 +1333,5 @@ func (c *CronTask) compareWithPreviousSnapshot(
|
||||
newCount = len(presentSnapshots)
|
||||
}
|
||||
|
||||
return prevTableName, newCount, missingCount
|
||||
return prevTableName, newCount, missingCount, prevTableTouched
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user