improve logging for pro-rata

db/helpers.go

@@ -492,12 +492,16 @@ CREATE TABLE IF NOT EXISTS vm_lifecycle_cache (
 }
 
 // UpsertVmLifecycleCache updates first/last seen info for a VM.
-func UpsertVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID, vmUUID, name, cluster string, seen time.Time) error {
+func UpsertVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID, vmUUID, name, cluster string, seen time.Time, creation sql.NullInt64) error {
 	if err := EnsureVmLifecycleCache(ctx, dbConn); err != nil {
 		return err
 	}
 	driver := strings.ToLower(dbConn.DriverName())
 	bindType := sqlx.BindType(driver)
+	firstSeen := seen.Unix()
+	if creation.Valid && creation.Int64 > 0 && creation.Int64 < firstSeen {
+		firstSeen = creation.Int64
+	}
 	query := `
 INSERT INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","Name","Cluster","FirstSeen","LastSeen")
 VALUES (?, ?, ?, ?, ?, ?, ?)
@@ -505,11 +509,16 @@ ON CONFLICT ("Vcenter","VmId","VmUuid") DO UPDATE SET
   "Name"=EXCLUDED."Name",
   "Cluster"=EXCLUDED."Cluster",
   "LastSeen"=EXCLUDED."LastSeen",
-  "FirstSeen"=COALESCE(vm_lifecycle_cache."FirstSeen", EXCLUDED."FirstSeen"),
+  "FirstSeen"=CASE
+    WHEN vm_lifecycle_cache."FirstSeen" IS NULL OR vm_lifecycle_cache."FirstSeen" = 0 THEN EXCLUDED."FirstSeen"
+    WHEN EXCLUDED."FirstSeen" IS NOT NULL AND EXCLUDED."FirstSeen" > 0 AND EXCLUDED."FirstSeen" < vm_lifecycle_cache."FirstSeen"
+      THEN EXCLUDED."FirstSeen"
+    ELSE vm_lifecycle_cache."FirstSeen"
+  END,
   "DeletedAt"=NULL
 `
 	query = sqlx.Rebind(bindType, query)
-	args := []interface{}{vcenter, vmID, vmUUID, name, cluster, seen.Unix(), seen.Unix()}
+	args := []interface{}{vcenter, vmID, vmUUID, name, cluster, firstSeen, seen.Unix()}
 	_, err := dbConn.ExecContext(ctx, query, args...)
 	if err != nil {
 		slog.Warn("lifecycle upsert exec failed", "vcenter", vcenter, "vm_id", vmID, "vm_uuid", vmUUID, "driver", driver, "args_len", len(args), "args", fmt.Sprint(args), "query", strings.TrimSpace(query), "error", err)

internal/report/snapshots.go

@@ -1164,6 +1164,7 @@ WITH base AS (
     "VmId",
     "VmUuid",
     "Name",
+    "Vcenter",
     "VcpuCount",
     "RamGB",
     LOWER(COALESCE("ResourcePool", '')) AS pool,
@@ -1184,14 +1185,25 @@ agg AS (
   FROM base
   GROUP BY vm_key
 ),
+lifecycle AS (
+  SELECT
+    (COALESCE(NULLIF("VmId", ''), NULLIF("VmUuid", ''), NULLIF("Name", ''), 'unknown') || '|' || COALESCE("Vcenter", '')) AS vm_key,
+    MIN(NULLIF("FirstSeen", 0)) AS first_seen,
+    MIN(NULLIF("DeletedAt", 0)) AS deleted_at
+  FROM vm_lifecycle_cache
+  GROUP BY vm_key
+),
 agg_presence AS (
   SELECT
-    vm_key,
-    "VcpuCount",
-    "RamGB",
-    pool,
+    agg.vm_key,
+    agg."VcpuCount",
+    agg."RamGB",
+    agg.pool,
+    COALESCE(agg.creation_time, lifecycle.first_seen) AS creation_time,
+    COALESCE(lifecycle.deleted_at, agg.deletion_time) AS deletion_time,
     %s AS presence
   FROM agg
+  LEFT JOIN lifecycle ON lifecycle.vm_key = agg.vm_key
 ),
 diag AS (
   SELECT
@@ -1205,6 +1217,16 @@ diag AS (
     COALESCE(SUM(CASE WHEN presence < 0 THEN 1 ELSE 0 END), 0) AS presence_under_zero,
     COALESCE(SUM(presence), 0) AS base_presence_sum
   FROM base
+),
+agg_diag AS (
+  SELECT
+    COUNT(*) AS agg_count,
+    COALESCE(SUM(CASE WHEN creation_time IS NULL OR creation_time = 0 THEN 1 ELSE 0 END), 0) AS missing_creation,
+    COALESCE(SUM(CASE WHEN deletion_time IS NULL OR deletion_time = 0 THEN 1 ELSE 0 END), 0) AS missing_deletion,
+    COALESCE(SUM(CASE WHEN creation_time > ? AND creation_time < ? THEN 1 ELSE 0 END), 0) AS created_in_interval,
+    COALESCE(SUM(CASE WHEN deletion_time > ? AND deletion_time < ? THEN 1 ELSE 0 END), 0) AS deleted_in_interval,
+    COALESCE(SUM(CASE WHEN presence > 0 AND presence < 1 THEN 1 ELSE 0 END), 0) AS partial_presence
+  FROM agg_presence
 )
 SELECT
   (SELECT COUNT(*) FROM agg_presence) AS vm_count,
@@ -1223,8 +1245,14 @@ SELECT
   diag.missing_name,
   diag.presence_over_one,
   diag.presence_under_zero,
-  diag.base_presence_sum
-FROM diag
+  diag.base_presence_sum,
+  agg_diag.agg_count,
+  agg_diag.missing_creation,
+  agg_diag.missing_deletion,
+  agg_diag.created_in_interval,
+  agg_diag.deleted_in_interval,
+  agg_diag.partial_presence
+FROM diag, agg_diag
 `, vmKeyExpr, overlapExpr, selected.TableName, templateExclusionFilter(), aggOverlapExpr)
 		query = dbConn.Rebind(query)
 		var row struct {
@@ -1245,6 +1273,12 @@ FROM diag
 			PresenceOverOne  int64   `db:"presence_over_one"`
 			PresenceUnderZero int64  `db:"presence_under_zero"`
 			BasePresenceSum  float64 `db:"base_presence_sum"`
+			AggCount         int64   `db:"agg_count"`
+			MissingCreation  int64   `db:"missing_creation"`
+			MissingDeletion  int64   `db:"missing_deletion"`
+			CreatedInInterval int64  `db:"created_in_interval"`
+			DeletedInInterval int64  `db:"deleted_in_interval"`
+			PartialPresence  int64   `db:"partial_presence"`
 		}
 		args := []interface{}{
 			hourEndUnix, hourEndUnix,
@@ -1257,6 +1291,8 @@ FROM diag
 			hourEndUnix, hourEndUnix,
 			hourStartUnix, hourStartUnix,
 			durationSeconds,
+			hourStartUnix, hourEndUnix,
+			hourStartUnix, hourEndUnix,
 		}
 		if err := dbConn.GetContext(ctx, &row, query, args...); err != nil {
 			return nil, err
@@ -1280,6 +1316,12 @@ FROM diag
 			"presence_over_one", row.PresenceOverOne,
 			"presence_under_zero", row.PresenceUnderZero,
 			"base_presence_sum", row.BasePresenceSum,
+			"agg_count", row.AggCount,
+			"missing_creation", row.MissingCreation,
+			"missing_deletion", row.MissingDeletion,
+			"created_in_interval", row.CreatedInInterval,
+			"deleted_in_interval", row.DeletedInInterval,
+			"partial_presence", row.PartialPresence,
 			"presence_ratio", row.PresenceRatio,
 			"vm_count", row.VmCount,
 		)

internal/tasks/inventorySnapshots.go

@@ -928,7 +928,7 @@ func (c *CronTask) buildPresentSnapshots(ctx context.Context, dbConn *sqlx.DB, v
 		if row.Cluster.Valid {
 			clusterName = row.Cluster.String
 		}
-		if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, row.VmId.String, row.VmUuid.String, row.Name, clusterName, startTime); err != nil {
+		if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, row.VmId.String, row.VmUuid.String, row.Name, clusterName, startTime, row.CreationTime); err != nil {
 			log.Warn("failed to upsert vm lifecycle cache", "vcenter", url, "vm_id", row.VmId, "vm_uuid", row.VmUuid, "name", row.Name, "error", err)
 		}
 		presentSnapshots[vm.Reference().Value] = row