From 93b5769145be602acf58a3185e351a03c4c37bd4 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Tue, 27 Jan 2026 21:40:41 +1100 Subject: [PATCH] improve logging for pro-rata --- db/helpers.go | 15 ++++++-- internal/report/snapshots.go | 54 ++++++++++++++++++++++++---- internal/tasks/inventorySnapshots.go | 2 +- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/db/helpers.go b/db/helpers.go index e5d1f88..a9b04fd 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -492,12 +492,16 @@ CREATE TABLE IF NOT EXISTS vm_lifecycle_cache ( } // UpsertVmLifecycleCache updates first/last seen info for a VM. -func UpsertVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID, vmUUID, name, cluster string, seen time.Time) error { +func UpsertVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID, vmUUID, name, cluster string, seen time.Time, creation sql.NullInt64) error { if err := EnsureVmLifecycleCache(ctx, dbConn); err != nil { return err } driver := strings.ToLower(dbConn.DriverName()) bindType := sqlx.BindType(driver) + firstSeen := seen.Unix() + if creation.Valid && creation.Int64 > 0 && creation.Int64 < firstSeen { + firstSeen = creation.Int64 + } query := ` INSERT INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","Name","Cluster","FirstSeen","LastSeen") VALUES (?, ?, ?, ?, ?, ?, ?) @@ -505,11 +509,16 @@ ON CONFLICT ("Vcenter","VmId","VmUuid") DO UPDATE SET "Name"=EXCLUDED."Name", "Cluster"=EXCLUDED."Cluster", "LastSeen"=EXCLUDED."LastSeen", - "FirstSeen"=COALESCE(vm_lifecycle_cache."FirstSeen", EXCLUDED."FirstSeen"), + "FirstSeen"=CASE + WHEN vm_lifecycle_cache."FirstSeen" IS NULL OR vm_lifecycle_cache."FirstSeen" = 0 THEN EXCLUDED."FirstSeen" + WHEN EXCLUDED."FirstSeen" IS NOT NULL AND EXCLUDED."FirstSeen" > 0 AND EXCLUDED."FirstSeen" < vm_lifecycle_cache."FirstSeen" + THEN EXCLUDED."FirstSeen" + ELSE vm_lifecycle_cache."FirstSeen" + END, "DeletedAt"=NULL ` query = sqlx.Rebind(bindType, query) - args := []interface{}{vcenter, vmID, vmUUID, name, cluster, seen.Unix(), seen.Unix()} + args := []interface{}{vcenter, vmID, vmUUID, name, cluster, firstSeen, seen.Unix()} _, err := dbConn.ExecContext(ctx, query, args...) if err != nil { slog.Warn("lifecycle upsert exec failed", "vcenter", vcenter, "vm_id", vmID, "vm_uuid", vmUUID, "driver", driver, "args_len", len(args), "args", fmt.Sprint(args), "query", strings.TrimSpace(query), "error", err) diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index 6537430..7e4d354 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -1164,6 +1164,7 @@ WITH base AS ( "VmId", "VmUuid", "Name", + "Vcenter", "VcpuCount", "RamGB", LOWER(COALESCE("ResourcePool", '')) AS pool, @@ -1184,14 +1185,25 @@ agg AS ( FROM base GROUP BY vm_key ), +lifecycle AS ( + SELECT + (COALESCE(NULLIF("VmId", ''), NULLIF("VmUuid", ''), NULLIF("Name", ''), 'unknown') || '|' || COALESCE("Vcenter", '')) AS vm_key, + MIN(NULLIF("FirstSeen", 0)) AS first_seen, + MIN(NULLIF("DeletedAt", 0)) AS deleted_at + FROM vm_lifecycle_cache + GROUP BY vm_key +), agg_presence AS ( SELECT - vm_key, - "VcpuCount", - "RamGB", - pool, + agg.vm_key, + agg."VcpuCount", + agg."RamGB", + agg.pool, + COALESCE(agg.creation_time, lifecycle.first_seen) AS creation_time, + COALESCE(lifecycle.deleted_at, agg.deletion_time) AS deletion_time, %s AS presence FROM agg + LEFT JOIN lifecycle ON lifecycle.vm_key = agg.vm_key ), diag AS ( SELECT @@ -1205,6 +1217,16 @@ diag AS ( COALESCE(SUM(CASE WHEN presence < 0 THEN 1 ELSE 0 END), 0) AS presence_under_zero, COALESCE(SUM(presence), 0) AS base_presence_sum FROM base +), +agg_diag AS ( + SELECT + COUNT(*) AS agg_count, + COALESCE(SUM(CASE WHEN creation_time IS NULL OR creation_time = 0 THEN 1 ELSE 0 END), 0) AS missing_creation, + COALESCE(SUM(CASE WHEN deletion_time IS NULL OR deletion_time = 0 THEN 1 ELSE 0 END), 0) AS missing_deletion, + COALESCE(SUM(CASE WHEN creation_time > ? AND creation_time < ? THEN 1 ELSE 0 END), 0) AS created_in_interval, + COALESCE(SUM(CASE WHEN deletion_time > ? AND deletion_time < ? THEN 1 ELSE 0 END), 0) AS deleted_in_interval, + COALESCE(SUM(CASE WHEN presence > 0 AND presence < 1 THEN 1 ELSE 0 END), 0) AS partial_presence + FROM agg_presence ) SELECT (SELECT COUNT(*) FROM agg_presence) AS vm_count, @@ -1223,8 +1245,14 @@ SELECT diag.missing_name, diag.presence_over_one, diag.presence_under_zero, - diag.base_presence_sum -FROM diag + diag.base_presence_sum, + agg_diag.agg_count, + agg_diag.missing_creation, + agg_diag.missing_deletion, + agg_diag.created_in_interval, + agg_diag.deleted_in_interval, + agg_diag.partial_presence +FROM diag, agg_diag `, vmKeyExpr, overlapExpr, selected.TableName, templateExclusionFilter(), aggOverlapExpr) query = dbConn.Rebind(query) var row struct { @@ -1245,6 +1273,12 @@ FROM diag PresenceOverOne int64 `db:"presence_over_one"` PresenceUnderZero int64 `db:"presence_under_zero"` BasePresenceSum float64 `db:"base_presence_sum"` + AggCount int64 `db:"agg_count"` + MissingCreation int64 `db:"missing_creation"` + MissingDeletion int64 `db:"missing_deletion"` + CreatedInInterval int64 `db:"created_in_interval"` + DeletedInInterval int64 `db:"deleted_in_interval"` + PartialPresence int64 `db:"partial_presence"` } args := []interface{}{ hourEndUnix, hourEndUnix, @@ -1257,6 +1291,8 @@ FROM diag hourEndUnix, hourEndUnix, hourStartUnix, hourStartUnix, durationSeconds, + hourStartUnix, hourEndUnix, + hourStartUnix, hourEndUnix, } if err := dbConn.GetContext(ctx, &row, query, args...); err != nil { return nil, err @@ -1280,6 +1316,12 @@ FROM diag "presence_over_one", row.PresenceOverOne, "presence_under_zero", row.PresenceUnderZero, "base_presence_sum", row.BasePresenceSum, + "agg_count", row.AggCount, + "missing_creation", row.MissingCreation, + "missing_deletion", row.MissingDeletion, + "created_in_interval", row.CreatedInInterval, + "deleted_in_interval", row.DeletedInInterval, + "partial_presence", row.PartialPresence, "presence_ratio", row.PresenceRatio, "vm_count", row.VmCount, ) diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index f3c3cbf..9f86dfd 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -928,7 +928,7 @@ func (c *CronTask) buildPresentSnapshots(ctx context.Context, dbConn *sqlx.DB, v if row.Cluster.Valid { clusterName = row.Cluster.String } - if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, row.VmId.String, row.VmUuid.String, row.Name, clusterName, startTime); err != nil { + if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, row.VmId.String, row.VmUuid.String, row.Name, clusterName, startTime, row.CreationTime); err != nil { log.Warn("failed to upsert vm lifecycle cache", "vcenter", url, "vm_id", row.VmId, "vm_uuid", row.VmUuid, "name", row.Name, "error", err) } presentSnapshots[vm.Reference().Value] = row