diff --git a/components/views/vm_trace.templ b/components/views/vm_trace.templ index 362cfa7..8277dbe 100644 --- a/components/views/vm_trace.templ +++ b/components/views/vm_trace.templ @@ -35,7 +35,7 @@ type VmTraceChart struct { YTicks []ChartTick } -templ VmTracePage(query string, display_query string, vm_id string, vm_uuid string, vm_name string, creationLabel string, deletionLabel string, entries []VmTraceEntry, chart VmTraceChart) { +templ VmTracePage(query string, display_query string, vm_id string, vm_uuid string, vm_name string, creationLabel string, deletionLabel string, creationApprox bool, entries []VmTraceEntry, chart VmTraceChart) { @core.Header() @@ -124,6 +124,9 @@ templ VmTracePage(query string, display_query string, vm_id string, vm_uuid stri

Creation time

{creationLabel}

+ if creationApprox { +

Approximate (earliest snapshot)

+ }

Deletion time

diff --git a/components/views/vm_trace_templ.go b/components/views/vm_trace_templ.go index 2900849..60d1328 100644 --- a/components/views/vm_trace_templ.go +++ b/components/views/vm_trace_templ.go @@ -43,7 +43,7 @@ type VmTraceChart struct { YTicks []ChartTick } -func VmTracePage(query string, display_query string, vm_id string, vm_uuid string, vm_name string, creationLabel string, deletionLabel string, entries []VmTraceEntry, chart VmTraceChart) templ.Component { +func VmTracePage(query string, display_query string, vm_id string, vm_uuid string, vm_name string, creationLabel string, deletionLabel string, creationApprox bool, entries []VmTraceEntry, chart VmTraceChart) templ.Component { return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) { templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil { @@ -560,73 +560,57 @@ func VmTracePage(query string, display_query string, vm_id string, vm_uuid strin if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "

Deletion time

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + if creationApprox { + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "

Approximate (earliest snapshot)

") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "

Deletion time

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var36 string templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(deletionLabel) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 130, Col: 76} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 133, Col: 76} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "

SnapshotVM NameVmIdVmUuidVcenterResource PoolvCPUsRAM (GB)Disk
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } for _, e := range entries { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "
SnapshotVM NameVmIdVmUuidVcenterResource PoolvCPUsRAM (GB)Disk
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var37 string templ_7745c5c3_Var37, templ_7745c5c3_Err = templ.JoinStringErrs(e.Snapshot) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 151, Col: 25} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 154, Col: 25} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var37)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var38 string - templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(e.Name) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 152, Col: 21} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var39 string - templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(e.VmId) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 153, Col: 21} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var40 string - templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(e.VmUuid) + var templ_7745c5c3_Var38 string + templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(e.Name) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 154, Col: 23} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 155, Col: 21} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var40)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -634,12 +618,12 @@ func VmTracePage(query string, display_query string, vm_id string, vm_uuid strin if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var41 string - templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(e.Vcenter) + var templ_7745c5c3_Var39 string + templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(e.VmId) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 155, Col: 24} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 156, Col: 21} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -647,60 +631,86 @@ func VmTracePage(query string, display_query string, vm_id string, vm_uuid strin if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } + var templ_7745c5c3_Var40 string + templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(e.VmUuid) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 157, Col: 23} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var40)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var41 string + templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(e.Vcenter) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 158, Col: 24} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } var templ_7745c5c3_Var42 string templ_7745c5c3_Var42, templ_7745c5c3_Err = templ.JoinStringErrs(e.ResourcePool) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 156, Col: 29} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 159, Col: 29} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var42)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var43 string - templ_7745c5c3_Var43, templ_7745c5c3_Err = templ.JoinStringErrs(e.VcpuCount) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 157, Col: 45} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var43)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "") - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } - var templ_7745c5c3_Var44 string - templ_7745c5c3_Var44, templ_7745c5c3_Err = templ.JoinStringErrs(e.RamGB) - if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 158, Col: 41} - } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var44)) - if templ_7745c5c3_Err != nil { - return templ_7745c5c3_Err - } templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } + var templ_7745c5c3_Var43 string + templ_7745c5c3_Var43, templ_7745c5c3_Err = templ.JoinStringErrs(e.VcpuCount) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 160, Col: 45} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var43)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var44 string + templ_7745c5c3_Var44, templ_7745c5c3_Err = templ.JoinStringErrs(e.RamGB) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 161, Col: 41} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var44)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } var templ_7745c5c3_Var45 string templ_7745c5c3_Var45, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%.1f", e.ProvisionedDisk)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 159, Col: 72} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/vm_trace.templ`, Line: 162, Col: 72} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var45)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -708,7 +718,7 @@ func VmTracePage(query string, display_query string, vm_id string, vm_uuid strin if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/db/helpers.go b/db/helpers.go index 71d59ef..6360e05 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -504,6 +504,133 @@ VALUES (?,?,?,?,?,?) return err } +// UpsertVmDailyRollup writes/updates a daily rollup row. +func UpsertVmDailyRollup(ctx context.Context, dbConn *sqlx.DB, day int64, v VmDailyRollupRow) error { + if err := EnsureVmDailyRollup(ctx, dbConn); err != nil { + return err + } + driver := strings.ToLower(dbConn.DriverName()) + query := ` +INSERT INTO vm_daily_rollup ( + "Date","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","SamplesPresent","TotalSamples", + "SumVcpu","SumRam","SumDisk","TinHits","BronzeHits","SilverHits","GoldHits", + "LastResourcePool","LastDatacenter","LastCluster","LastFolder", + "LastProvisionedDisk","LastVcpuCount","LastRamGB","IsTemplate","PoweredOn","SrmPlaceholder" +) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26) +ON CONFLICT ("Date","Vcenter","VmId","VmUuid") DO UPDATE SET + "CreationTime"=LEAST(COALESCE(vm_daily_rollup."CreationTime", $6), COALESCE($6, vm_daily_rollup."CreationTime")), + "DeletionTime"=CASE + WHEN vm_daily_rollup."DeletionTime" IS NULL OR vm_daily_rollup."DeletionTime"=0 THEN $7 + WHEN $7 IS NOT NULL AND $7 > 0 AND $7 < vm_daily_rollup."DeletionTime" THEN $7 + ELSE vm_daily_rollup."DeletionTime" END, + "SamplesPresent"=$8, + "TotalSamples"=$9, + "SumVcpu"=$10, + "SumRam"=$11, + "SumDisk"=$12, + "TinHits"=$13, + "BronzeHits"=$14, + "SilverHits"=$15, + "GoldHits"=$16, + "LastResourcePool"=$17, + "LastDatacenter"=$18, + "LastCluster"=$19, + "LastFolder"=$20, + "LastProvisionedDisk"=$21, + "LastVcpuCount"=$22, + "LastRamGB"=$23, + "IsTemplate"=$24, + "PoweredOn"=$25, + "SrmPlaceholder"=$26 +` + args := []interface{}{ + day, v.Vcenter, v.VmId, v.VmUuid, v.Name, v.CreationTime, v.DeletionTime, v.SamplesPresent, v.TotalSamples, + v.SumVcpu, v.SumRam, v.SumDisk, v.TinHits, v.BronzeHits, v.SilverHits, v.GoldHits, + v.LastResourcePool, v.LastDatacenter, v.LastCluster, v.LastFolder, v.LastProvisionedDisk, v.LastVcpuCount, v.LastRamGB, v.IsTemplate, v.PoweredOn, v.SrmPlaceholder, + } + if driver == "sqlite" { + query = ` +INSERT OR REPLACE INTO vm_daily_rollup ( + "Date","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","SamplesPresent","TotalSamples", + "SumVcpu","SumRam","SumDisk","TinHits","BronzeHits","SilverHits","GoldHits", + "LastResourcePool","LastDatacenter","LastCluster","LastFolder", + "LastProvisionedDisk","LastVcpuCount","LastRamGB","IsTemplate","PoweredOn","SrmPlaceholder" +) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +` + } + _, err := dbConn.ExecContext(ctx, query, args...) + return err +} + +// VmDailyRollupRow represents the per-day cached aggregation. +type VmDailyRollupRow struct { + Vcenter string + VmId string + VmUuid string + Name string + CreationTime int64 + DeletionTime int64 + SamplesPresent int64 + TotalSamples int64 + SumVcpu float64 + SumRam float64 + SumDisk float64 + TinHits int64 + BronzeHits int64 + SilverHits int64 + GoldHits int64 + LastResourcePool string + LastDatacenter string + LastCluster string + LastFolder string + LastProvisionedDisk float64 + LastVcpuCount int64 + LastRamGB int64 + IsTemplate string + PoweredOn string + SrmPlaceholder string +} + +// EnsureVmDailyRollup creates the per-day cache used by monthly aggregation. +func EnsureVmDailyRollup(ctx context.Context, dbConn *sqlx.DB) error { + ddl := ` +CREATE TABLE IF NOT EXISTS vm_daily_rollup ( + "Date" BIGINT NOT NULL, + "Vcenter" TEXT NOT NULL, + "VmId" TEXT, + "VmUuid" TEXT, + "Name" TEXT, + "CreationTime" BIGINT, + "DeletionTime" BIGINT, + "SamplesPresent" BIGINT, + "TotalSamples" BIGINT, + "SumVcpu" BIGINT, + "SumRam" BIGINT, + "SumDisk" REAL, + "TinHits" BIGINT, + "BronzeHits" BIGINT, + "SilverHits" BIGINT, + "GoldHits" BIGINT, + "LastResourcePool" TEXT, + "LastDatacenter" TEXT, + "LastCluster" TEXT, + "LastFolder" TEXT, + "LastProvisionedDisk" REAL, + "LastVcpuCount" BIGINT, + "LastRamGB" BIGINT, + "IsTemplate" TEXT, + "PoweredOn" TEXT, + "SrmPlaceholder" TEXT, + PRIMARY KEY ("Date","Vcenter","VmId","VmUuid") +);` + if _, err := execLog(ctx, dbConn, ddl); err != nil { + return err + } + _, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_daily_rollup_date_idx ON vm_daily_rollup ("Date")`) + _, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_daily_rollup_vcenter_date_idx ON vm_daily_rollup ("Vcenter","Date")`) + return nil +} + // EnsureVmIdentityTables creates the identity and rename audit tables. func EnsureVmIdentityTables(ctx context.Context, dbConn *sqlx.DB) error { driver := strings.ToLower(dbConn.DriverName()) @@ -844,10 +971,11 @@ type VmTraceRow struct { // VmLifecycle captures observed lifecycle times from hourly snapshots. type VmLifecycle struct { - CreationTime int64 - FirstSeen int64 - LastSeen int64 - DeletionTime int64 + CreationTime int64 + CreationApprox bool + FirstSeen int64 + LastSeen int64 + DeletionTime int64 } // FetchVmTrace returns combined hourly snapshot records for a VM (by id/uuid/name) ordered by snapshot time. @@ -928,6 +1056,7 @@ ORDER BY snapshot_time driver := strings.ToLower(dbConn.DriverName()) minCreation := int64(0) + consecutiveMissing := 0 for _, t := range tables { if err := ValidateTableName(t.TableName); err != nil { continue @@ -956,20 +1085,29 @@ WHERE ("VmId" = ? OR "VmUuid" = ? OR lower("Name") = lower(?)) lifecycle.FirstSeen = t.SnapshotTime } lifecycle.LastSeen = t.SnapshotTime + consecutiveMissing = 0 if probe.MinCreation.Valid { if minCreation == 0 || probe.MinCreation.Int64 < minCreation { minCreation = probe.MinCreation.Int64 } } } else if lifecycle.LastSeen > 0 && lifecycle.DeletionTime == 0 && t.SnapshotTime > lifecycle.LastSeen { - lifecycle.DeletionTime = t.SnapshotTime - break + consecutiveMissing++ + if consecutiveMissing >= 2 { + lifecycle.DeletionTime = t.SnapshotTime + break + } + } else { + // reset if we haven't seen the VM yet + consecutiveMissing = 0 } } if minCreation > 0 { lifecycle.CreationTime = minCreation + lifecycle.CreationApprox = false } else if lifecycle.FirstSeen > 0 { lifecycle.CreationTime = lifecycle.FirstSeen + lifecycle.CreationApprox = true } return lifecycle, nil } @@ -1208,8 +1346,8 @@ SELECT CASE WHEN totals.total_samples > 0 THEN 1.0 * agg.sum_ram / totals.total_samples ELSE NULL END AS "AvgRamGB", - CASE WHEN totals.total_samples > 0 - THEN 1.0 * agg.sum_disk / totals.total_samples + CASE WHEN agg.samples_present > 0 + THEN 1.0 * agg.sum_disk / agg.samples_present ELSE NULL END AS "AvgProvisionedDisk", CASE WHEN totals.total_samples > 0 THEN 1.0 * agg.samples_present / totals.total_samples diff --git a/internal/tasks/dailyAggregate.go b/internal/tasks/dailyAggregate.go index 010ace8..ffec5fb 100644 --- a/internal/tasks/dailyAggregate.go +++ b/internal/tasks/dailyAggregate.go @@ -280,6 +280,11 @@ func (c *CronTask) aggregateDailySummaryGo(ctx context.Context, dayStart, dayEnd return err } + // Persist rollup cache for monthly aggregation. + if err := c.persistDailyRollup(ctx, dayStart.Unix(), aggMap, totalSamples); err != nil { + c.Logger.Warn("failed to persist daily rollup cache", "error", err, "date", dayStart.Format("2006-01-02")) + } + // Refine lifecycle with existing SQL helper to pick up first-after deletions. if err := db.RefineCreationDeletionFromUnion(ctx, dbConn, summaryTable, unionQuery); err != nil { c.Logger.Warn("failed to refine creation/deletion times", "error", err, "table", summaryTable) @@ -727,3 +732,44 @@ func btoi(b bool) int64 { } return 0 } + +// persistDailyRollup stores per-day aggregates into vm_daily_rollup to speed monthly aggregation. +func (c *CronTask) persistDailyRollup(ctx context.Context, dayUnix int64, agg map[dailyAggKey]*dailyAggVal, totalSamples int) error { + dbConn := c.Database.DB() + for _, v := range agg { + if strings.EqualFold(strings.TrimSpace(v.isTemplate), "true") || v.isTemplate == "1" { + continue + } + row := db.VmDailyRollupRow{ + Vcenter: v.key.Vcenter, + VmId: v.key.VmId, + VmUuid: v.key.VmUuid, + Name: v.key.Name, + CreationTime: v.creation, + DeletionTime: v.deletion, + SamplesPresent: v.samples, + TotalSamples: int64(totalSamples), + SumVcpu: float64(v.sumVcpu), + SumRam: float64(v.sumRam), + SumDisk: v.sumDisk, + TinHits: v.tinHits, + BronzeHits: v.bronzeHits, + SilverHits: v.silverHits, + GoldHits: v.goldHits, + LastResourcePool: v.resourcePool, + LastDatacenter: v.datacenter, + LastCluster: v.cluster, + LastFolder: v.folder, + LastProvisionedDisk: v.lastDisk, + LastVcpuCount: v.lastVcpu, + LastRamGB: v.lastRam, + IsTemplate: v.isTemplate, + PoweredOn: v.poweredOn, + SrmPlaceholder: v.srmPlaceholder, + } + if err := db.UpsertVmDailyRollup(ctx, dbConn, dayUnix, row); err != nil { + return err + } + } + return nil +} diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index 0a32e0f..0e74ee1 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -102,7 +102,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo if err != nil { return err } - minIntervalSeconds := intWithDefault(c.Settings.Values.Settings.VcenterInventorySnapshotSeconds, 3600) + minIntervalSeconds := intWithDefault(c.Settings.Values.Settings.VcenterInventorySnapshotSeconds, 3600) / 2 if !lastSnapshot.IsZero() && startTime.Sub(lastSnapshot) < time.Duration(minIntervalSeconds)*time.Second { c.Logger.Info("Skipping hourly snapshot, last snapshot too recent", "last_snapshot", lastSnapshot, @@ -882,8 +882,12 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim return fmt.Errorf("unable to connect to vcenter: %w", err) } defer func() { - if err := vc.Logout(); err != nil { + logCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := vc.Logout(logCtx); err != nil { c.Logger.Warn("vcenter logout failed", "url", url, "error", err) + } else { + c.Logger.Debug("vcenter logout succeeded", "url", url) } }() diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index 70b4cda..bb97006 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -110,7 +110,7 @@ func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) erro } } c.Logger.Debug("Finished checking vcenter", "url", url) - vc.Logout() + _ = vc.Logout(ctx) } c.Logger.Debug("Finished polling vcenters") diff --git a/internal/tasks/monthlyAggregate.go b/internal/tasks/monthlyAggregate.go index c3554c1..b02df3e 100644 --- a/internal/tasks/monthlyAggregate.go +++ b/internal/tasks/monthlyAggregate.go @@ -173,6 +173,14 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo if err != nil { return err } + if len(aggMap) == 0 { + cacheAgg, cacheErr := c.scanDailyRollup(ctx, monthStart, monthEnd) + if cacheErr == nil && len(cacheAgg) > 0 { + aggMap = cacheAgg + } else if cacheErr != nil { + c.Logger.Warn("failed to read daily rollup cache; using table scan", "error", cacheErr) + } + } if len(aggMap) == 0 { return fmt.Errorf("no VM records aggregated for %s", monthStart.Format("2006-01")) } @@ -439,6 +447,89 @@ FROM %s return result, rows.Err() } +// scanDailyRollup aggregates monthly data from vm_daily_rollup cache. +func (c *CronTask) scanDailyRollup(ctx context.Context, start, end time.Time) (map[monthlyAggKey]*monthlyAggVal, error) { + dbConn := c.Database.DB() + if !db.TableExists(ctx, dbConn, "vm_daily_rollup") { + return map[monthlyAggKey]*monthlyAggVal{}, nil + } + query := ` +SELECT + "Date","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime", + "SamplesPresent","TotalSamples","SumVcpu","SumRam","SumDisk", + "TinHits","BronzeHits","SilverHits","GoldHits", + "LastResourcePool","LastDatacenter","LastCluster","LastFolder", + "LastProvisionedDisk","LastVcpuCount","LastRamGB","IsTemplate","PoweredOn","SrmPlaceholder" +FROM vm_daily_rollup +WHERE "Date" >= ? AND "Date" < ? +` + bind := dbConn.Rebind(query) + rows, err := dbConn.QueryxContext(ctx, bind, start.Unix(), end.Unix()) + if err != nil { + return nil, err + } + defer rows.Close() + + agg := make(map[monthlyAggKey]*monthlyAggVal, 512) + for rows.Next() { + var ( + date sql.NullInt64 + vcenter, vmId, vmUuid, name string + creation, deletion sql.NullInt64 + samplesPresent, totalSamples sql.NullInt64 + sumVcpu, sumRam, sumDisk sql.NullFloat64 + tinHits, bronzeHits, silverHits, goldHits sql.NullInt64 + lastPool, lastDc, lastCluster, lastFolder sql.NullString + lastDisk, lastVcpu, lastRam sql.NullFloat64 + isTemplate, poweredOn, srmPlaceholder sql.NullString + ) + if err := rows.Scan( + &date, &vcenter, &vmId, &vmUuid, &name, &creation, &deletion, + &samplesPresent, &totalSamples, &sumVcpu, &sumRam, &sumDisk, + &tinHits, &bronzeHits, &silverHits, &goldHits, + &lastPool, &lastDc, &lastCluster, &lastFolder, + &lastDisk, &lastVcpu, &lastRam, &isTemplate, &poweredOn, &srmPlaceholder, + ); err != nil { + continue + } + if strings.EqualFold(strings.TrimSpace(isTemplate.String), "true") || isTemplate.String == "1" { + continue + } + key := monthlyAggKey{Vcenter: vcenter, VmId: vmId, VmUuid: vmUuid, Name: name} + val := &monthlyAggVal{ + key: key, + resourcePool: lastPool.String, + datacenter: lastDc.String, + cluster: lastCluster.String, + folder: lastFolder.String, + isTemplate: isTemplate.String, + poweredOn: poweredOn.String, + srmPlaceholder: srmPlaceholder.String, + provisioned: lastDisk.Float64, + vcpuCount: int64(lastVcpu.Float64), + ramGB: int64(lastRam.Float64), + creation: creation.Int64, + deletion: deletion.Int64, + lastSnapshot: time.Unix(date.Int64, 0), + samplesPresent: samplesPresent.Int64, + totalSamples: float64(totalSamples.Int64), + sumVcpu: sumVcpu.Float64, + sumRam: sumRam.Float64, + sumDisk: sumDisk.Float64, + tinWeighted: float64(tinHits.Int64), + bronzeWeighted: float64(bronzeHits.Int64), + silverWeighted: float64(silverHits.Int64), + goldWeighted: float64(goldHits.Int64), + } + if existing, ok := agg[key]; ok { + mergeMonthlyAgg(existing, val) + } else { + agg[key] = val + } + } + return agg, rows.Err() +} + func (c *CronTask) insertMonthlyAggregates(ctx context.Context, summaryTable string, aggMap map[monthlyAggKey]*monthlyAggVal) error { dbConn := c.Database.DB() columns := []string{ diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index 4a29dce..7985d24 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -165,10 +165,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { poweredOn = "TRUE" } - err = vc.Logout() - if err != nil { - c.Logger.Error("unable to logout of vcenter", "error", err) - } + _ = vc.Logout(ctx) if foundVm { c.Logger.Debug("Adding to Inventory table", "vm_name", evt.VmName.String, "vcpus", numVcpus, "ram", numRam, "dc", evt.DatacenterId.String) diff --git a/internal/vcenter/vcenter.go b/internal/vcenter/vcenter.go index c633d87..aa02245 100644 --- a/internal/vcenter/vcenter.go +++ b/internal/vcenter/vcenter.go @@ -36,6 +36,15 @@ type VmProperties struct { ResourcePool string } +var clientUserAgent = "vCTP" + +// SetUserAgent customizes the User-Agent used when talking to vCenter. +func SetUserAgent(ua string) { + if strings.TrimSpace(ua) != "" { + clientUserAgent = ua + } +} + type HostLookup struct { Cluster string Datacenter string @@ -87,6 +96,9 @@ func (v *Vcenter) Login(vUrl string) error { v.Logger.Error("Unable to connect to vCenter", "error", err) return fmt.Errorf("unable to connect to vCenter : %s", err) } + if clientUserAgent != "" { + c.Client.UserAgent = clientUserAgent + } //defer c.Logout(v.ctx) @@ -97,22 +109,19 @@ func (v *Vcenter) Login(vUrl string) error { return nil } -func (v *Vcenter) Logout() error { - //v.Logger.Debug("vcenter logging out") - - if v.ctx == nil { +func (v *Vcenter) Logout(ctx context.Context) error { + if ctx == nil { + ctx = v.ctx + } + if ctx == nil { v.Logger.Warn("Nil context, unable to logout") return nil } - if v.client.Valid() { - //v.Logger.Debug("vcenter client is valid. Logging out") - return v.client.Logout(v.ctx) - } else { - v.Logger.Debug("vcenter client is not valid") - return nil + return v.client.Logout(ctx) } - + v.Logger.Debug("vcenter client is not valid") + return nil } func (v *Vcenter) GetAllVmReferences() ([]*object.VirtualMachine, error) { diff --git a/main.go b/main.go index ba2683c..8eb913b 100644 --- a/main.go +++ b/main.go @@ -155,6 +155,13 @@ func main() { os.Exit(1) } + // Set a recognizable User-Agent for vCenter sessions. + ua := "vCTP" + if sha1ver != "" { + ua = fmt.Sprintf("vCTP/%s", sha1ver) + } + vcenter.SetUserAgent(ua) + // Prepare the task scheduler c, err := gocron.NewScheduler() if err != nil { diff --git a/server/handler/vmModifyEvent.go b/server/handler/vmModifyEvent.go index 4802d69..966d4a5 100644 --- a/server/handler/vmModifyEvent.go +++ b/server/handler/vmModifyEvent.go @@ -404,10 +404,7 @@ func (h *Handler) calculateNewDiskSize(event models.CloudEventReceived) float64 } } - err = vc.Logout() - if err != nil { - h.Logger.Error("unable to logout of vcenter", "error", err) - } + _ = vc.Logout(context.Background()) h.Logger.Debug("Calculated new disk size", "value", diskSize) @@ -446,9 +443,7 @@ func (h *Handler) AddVmToInventory(evt models.CloudEventReceived, ctx context.Co if strings.HasPrefix(vmObject.Name, "vCLS-") { h.Logger.Info("Skipping internal vCLS VM", "vm_name", vmObject.Name) - if err := vc.Logout(); err != nil { - h.Logger.Error("unable to logout of vcenter", "error", err) - } + _ = vc.Logout(ctx) return 0, nil } @@ -522,10 +517,7 @@ func (h *Handler) AddVmToInventory(evt models.CloudEventReceived, ctx context.Co poweredOn = "TRUE" } - err = vc.Logout() - if err != nil { - h.Logger.Error("unable to logout of vcenter", "error", err) - } + _ = vc.Logout(ctx) if foundVm { e := evt.CloudEvent diff --git a/server/handler/vmTrace.go b/server/handler/vmTrace.go index 010f9e7..c0fe92e 100644 --- a/server/handler/vmTrace.go +++ b/server/handler/vmTrace.go @@ -35,6 +35,7 @@ func (h *Handler) VmTrace(w http.ResponseWriter, r *http.Request) { } creationLabel := "" deletionLabel := "" + creationApprox := false // Only fetch data when a query is provided; otherwise render empty page with form. if vmID != "" || vmUUID != "" || name != "" { @@ -79,9 +80,16 @@ func (h *Handler) VmTrace(w http.ResponseWriter, r *http.Request) { if len(entries) > 0 { if lifecycle.CreationTime > 0 { - creationLabel = time.Unix(lifecycle.CreationTime, 0).Local().Format("2006-01-02 15:04:05") + ts := time.Unix(lifecycle.CreationTime, 0).Local().Format("2006-01-02 15:04:05") + if lifecycle.CreationApprox { + creationLabel = fmt.Sprintf("%s (approx. earliest snapshot)", ts) + creationApprox = true + } else { + creationLabel = ts + } } else { creationLabel = time.Unix(entries[0].RawTime, 0).Local().Format("2006-01-02 15:04:05") + creationApprox = true } if lifecycle.DeletionTime > 0 { deletionLabel = time.Unix(lifecycle.DeletionTime, 0).Local().Format("2006-01-02 15:04:05") @@ -90,7 +98,7 @@ func (h *Handler) VmTrace(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := views.VmTracePage(queryLabel, displayQuery, vmID, vmUUID, name, creationLabel, deletionLabel, entries, chart).Render(ctx, w); err != nil { + if err := views.VmTracePage(queryLabel, displayQuery, vmID, vmUUID, name, creationLabel, deletionLabel, creationApprox, entries, chart).Render(ctx, w); err != nil { http.Error(w, "Failed to render template", http.StatusInternalServerError) } }