ensure we logout, fix aggregations

db/helpers.go

@@ -504,6 +504,133 @@ VALUES (?,?,?,?,?,?)
 	return err
 }
 
+// UpsertVmDailyRollup writes/updates a daily rollup row.
+func UpsertVmDailyRollup(ctx context.Context, dbConn *sqlx.DB, day int64, v VmDailyRollupRow) error {
+	if err := EnsureVmDailyRollup(ctx, dbConn); err != nil {
+		return err
+	}
+	driver := strings.ToLower(dbConn.DriverName())
+	query := `
+INSERT INTO vm_daily_rollup (
+  "Date","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","SamplesPresent","TotalSamples",
+  "SumVcpu","SumRam","SumDisk","TinHits","BronzeHits","SilverHits","GoldHits",
+  "LastResourcePool","LastDatacenter","LastCluster","LastFolder",
+  "LastProvisionedDisk","LastVcpuCount","LastRamGB","IsTemplate","PoweredOn","SrmPlaceholder"
+) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
+ON CONFLICT ("Date","Vcenter","VmId","VmUuid") DO UPDATE SET
+  "CreationTime"=LEAST(COALESCE(vm_daily_rollup."CreationTime", $6), COALESCE($6, vm_daily_rollup."CreationTime")),
+  "DeletionTime"=CASE
+    WHEN vm_daily_rollup."DeletionTime" IS NULL OR vm_daily_rollup."DeletionTime"=0 THEN $7
+    WHEN $7 IS NOT NULL AND $7 > 0 AND $7 < vm_daily_rollup."DeletionTime" THEN $7
+    ELSE vm_daily_rollup."DeletionTime" END,
+  "SamplesPresent"=$8,
+  "TotalSamples"=$9,
+  "SumVcpu"=$10,
+  "SumRam"=$11,
+  "SumDisk"=$12,
+  "TinHits"=$13,
+  "BronzeHits"=$14,
+  "SilverHits"=$15,
+  "GoldHits"=$16,
+  "LastResourcePool"=$17,
+  "LastDatacenter"=$18,
+  "LastCluster"=$19,
+  "LastFolder"=$20,
+  "LastProvisionedDisk"=$21,
+  "LastVcpuCount"=$22,
+  "LastRamGB"=$23,
+  "IsTemplate"=$24,
+  "PoweredOn"=$25,
+  "SrmPlaceholder"=$26
+`
+	args := []interface{}{
+		day, v.Vcenter, v.VmId, v.VmUuid, v.Name, v.CreationTime, v.DeletionTime, v.SamplesPresent, v.TotalSamples,
+		v.SumVcpu, v.SumRam, v.SumDisk, v.TinHits, v.BronzeHits, v.SilverHits, v.GoldHits,
+		v.LastResourcePool, v.LastDatacenter, v.LastCluster, v.LastFolder, v.LastProvisionedDisk, v.LastVcpuCount, v.LastRamGB, v.IsTemplate, v.PoweredOn, v.SrmPlaceholder,
+	}
+	if driver == "sqlite" {
+		query = `
+INSERT OR REPLACE INTO vm_daily_rollup (
+  "Date","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","SamplesPresent","TotalSamples",
+  "SumVcpu","SumRam","SumDisk","TinHits","BronzeHits","SilverHits","GoldHits",
+  "LastResourcePool","LastDatacenter","LastCluster","LastFolder",
+  "LastProvisionedDisk","LastVcpuCount","LastRamGB","IsTemplate","PoweredOn","SrmPlaceholder"
+) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+`
+	}
+	_, err := dbConn.ExecContext(ctx, query, args...)
+	return err
+}
+
+// VmDailyRollupRow represents the per-day cached aggregation.
+type VmDailyRollupRow struct {
+	Vcenter             string
+	VmId                string
+	VmUuid              string
+	Name                string
+	CreationTime        int64
+	DeletionTime        int64
+	SamplesPresent      int64
+	TotalSamples        int64
+	SumVcpu             float64
+	SumRam              float64
+	SumDisk             float64
+	TinHits             int64
+	BronzeHits          int64
+	SilverHits          int64
+	GoldHits            int64
+	LastResourcePool    string
+	LastDatacenter      string
+	LastCluster         string
+	LastFolder          string
+	LastProvisionedDisk float64
+	LastVcpuCount       int64
+	LastRamGB           int64
+	IsTemplate          string
+	PoweredOn           string
+	SrmPlaceholder      string
+}
+
+// EnsureVmDailyRollup creates the per-day cache used by monthly aggregation.
+func EnsureVmDailyRollup(ctx context.Context, dbConn *sqlx.DB) error {
+	ddl := `
+CREATE TABLE IF NOT EXISTS vm_daily_rollup (
+  "Date" BIGINT NOT NULL,
+  "Vcenter" TEXT NOT NULL,
+  "VmId" TEXT,
+  "VmUuid" TEXT,
+  "Name" TEXT,
+  "CreationTime" BIGINT,
+  "DeletionTime" BIGINT,
+  "SamplesPresent" BIGINT,
+  "TotalSamples" BIGINT,
+  "SumVcpu" BIGINT,
+  "SumRam" BIGINT,
+  "SumDisk" REAL,
+  "TinHits" BIGINT,
+  "BronzeHits" BIGINT,
+  "SilverHits" BIGINT,
+  "GoldHits" BIGINT,
+  "LastResourcePool" TEXT,
+  "LastDatacenter" TEXT,
+  "LastCluster" TEXT,
+  "LastFolder" TEXT,
+  "LastProvisionedDisk" REAL,
+  "LastVcpuCount" BIGINT,
+  "LastRamGB" BIGINT,
+  "IsTemplate" TEXT,
+  "PoweredOn" TEXT,
+  "SrmPlaceholder" TEXT,
+  PRIMARY KEY ("Date","Vcenter","VmId","VmUuid")
+);`
+	if _, err := execLog(ctx, dbConn, ddl); err != nil {
+		return err
+	}
+	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_daily_rollup_date_idx ON vm_daily_rollup ("Date")`)
+	_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_daily_rollup_vcenter_date_idx ON vm_daily_rollup ("Vcenter","Date")`)
+	return nil
+}
+
 // EnsureVmIdentityTables creates the identity and rename audit tables.
 func EnsureVmIdentityTables(ctx context.Context, dbConn *sqlx.DB) error {
 	driver := strings.ToLower(dbConn.DriverName())
@@ -844,10 +971,11 @@ type VmTraceRow struct {
 
 // VmLifecycle captures observed lifecycle times from hourly snapshots.
 type VmLifecycle struct {
-	CreationTime int64
-	FirstSeen    int64
-	LastSeen     int64
-	DeletionTime int64
+	CreationTime   int64
+	CreationApprox bool
+	FirstSeen      int64
+	LastSeen       int64
+	DeletionTime   int64
 }
 
 // FetchVmTrace returns combined hourly snapshot records for a VM (by id/uuid/name) ordered by snapshot time.
@@ -928,6 +1056,7 @@ ORDER BY snapshot_time
 	driver := strings.ToLower(dbConn.DriverName())
 
 	minCreation := int64(0)
+	consecutiveMissing := 0
 	for _, t := range tables {
 		if err := ValidateTableName(t.TableName); err != nil {
 			continue
@@ -956,20 +1085,29 @@ WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?))
 				lifecycle.FirstSeen = t.SnapshotTime
 			}
 			lifecycle.LastSeen = t.SnapshotTime
+			consecutiveMissing = 0
 			if probe.MinCreation.Valid {
 				if minCreation == 0 || probe.MinCreation.Int64 < minCreation {
 					minCreation = probe.MinCreation.Int64
 				}
 			}
 		} else if lifecycle.LastSeen > 0 && lifecycle.DeletionTime == 0 && t.SnapshotTime > lifecycle.LastSeen {
-			lifecycle.DeletionTime = t.SnapshotTime
-			break
+			consecutiveMissing++
+			if consecutiveMissing >= 2 {
+				lifecycle.DeletionTime = t.SnapshotTime
+				break
+			}
+		} else {
+			// reset if we haven't seen the VM yet
+			consecutiveMissing = 0
 		}
 	}
 	if minCreation > 0 {
 		lifecycle.CreationTime = minCreation
+		lifecycle.CreationApprox = false
 	} else if lifecycle.FirstSeen > 0 {
 		lifecycle.CreationTime = lifecycle.FirstSeen
+		lifecycle.CreationApprox = true
 	}
 	return lifecycle, nil
 }
@@ -1208,8 +1346,8 @@ SELECT
   CASE WHEN totals.total_samples > 0
        THEN 1.0 * agg.sum_ram / totals.total_samples
        ELSE NULL END AS "AvgRamGB",
-  CASE WHEN totals.total_samples > 0
-       THEN 1.0 * agg.sum_disk / totals.total_samples
+  CASE WHEN agg.samples_present > 0
+       THEN 1.0 * agg.sum_disk / agg.samples_present
        ELSE NULL END AS "AvgProvisionedDisk",
   CASE WHEN totals.total_samples > 0
        THEN 1.0 * agg.samples_present / totals.total_samples