nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix creationtime in aggregations
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-23 07:29:59 +11:00
parent 13adc159a2
commit 8a3481b966
9 changed files with 183 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
internal/tasks/dailyAggregate.go
View File

@@ -150,18 +150,16 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
return err
}
if applied, err := db.ApplyLifecycleCreationToSummary(ctx, dbConn, summaryTable); err != nil {
c.Logger.Warn("failed to apply lifecycle creation times to daily summary", "error", err, "table", summaryTable)
} else {
c.Logger.Info("Daily aggregation creation times", "source_lifecycle_cache", applied)
}
if applied, err := db.ApplyLifecycleDeletionToSummary(ctx, dbConn, summaryTable, dayStart.Unix(), dayEnd.Unix()); err != nil {
c.Logger.Warn("failed to apply lifecycle deletions to daily summary", "error", err, "table", summaryTable)
} else {
c.Logger.Info("Daily aggregation deletion times", "source_lifecycle_cache", applied)
}
// Backfill missing creation times to the start of the day for rows where vCenter had no creation info.
if _, err := dbConn.ExecContext(ctx,
`UPDATE `+summaryTable+` SET "CreationTime" = $1 WHERE "CreationTime" IS NULL OR "CreationTime" = 0`,
dayStart.Unix(),
); err != nil {
c.Logger.Warn("failed to normalize creation times for daily summary", "error", err, "table", summaryTable)
}
if err := db.RefineCreationDeletionFromUnion(ctx, dbConn, summaryTable, unionQuery); err != nil {
c.Logger.Warn("failed to refine creation/deletion times", "error", err, "table", summaryTable)
}
@@ -377,6 +375,12 @@ LIMIT 1
}
c.Logger.Debug("inserted daily aggregates", "table", summaryTable, "rows", len(aggMap), "total_samples", totalSamples)
if applied, err := db.ApplyLifecycleCreationToSummary(ctx, dbConn, summaryTable); err != nil {
c.Logger.Warn("failed to apply lifecycle creation times to daily summary (Go)", "error", err, "table", summaryTable)
} else {
c.Logger.Info("Daily aggregation creation times", "source_lifecycle_cache", applied)
}
// Persist rollup cache for monthly aggregation.
if err := c.persistDailyRollup(ctx, dayStart.Unix(), aggMap, totalSamples, totalSamplesByVcenter); err != nil {
c.Logger.Warn("failed to persist daily rollup cache", "error", err, "date", dayStart.Format("2006-01-02"))
@@ -489,9 +493,7 @@ WHERE "Vcenter" = ? AND "DeletedAt" IS NOT NULL AND "DeletedAt" > 0 AND "Deleted
missed++
continue
}
if target.deletion == 0 || deletedAt.Int64 < target.deletion {
target.deletion = deletedAt.Int64
}
target.deletion = deletedAt.Int64
applied++
}
rows.Close()
@@ -544,24 +546,24 @@ func (c *CronTask) applyInventoryDeletions(ctx context.Context, agg map[dailyAgg
continue
}
if ts, ok := byID[k.VmId]; ok {
if v.deletion == 0 || ts < v.deletion {
if v.deletion == 0 {
v.deletion = ts
totalApplied++
}
totalApplied++
continue
}
if ts, ok := byUUID[k.VmUuid]; ok {
if v.deletion == 0 || ts < v.deletion {
if v.deletion == 0 {
v.deletion = ts
totalApplied++
}
totalApplied++
continue
}
if ts, ok := byName[strings.ToLower(k.Name)]; ok {
if v.deletion == 0 || ts < v.deletion {
if v.deletion == 0 {
v.deletion = ts
totalApplied++
}
totalApplied++
}
}
}
@@ -884,9 +886,9 @@ INSERT INTO %s (
goldPct := 0.0
if total > 0 {
avgPresent = float64(v.samples) / total
avgVcpu = avgPresent * float64(v.lastVcpu)
avgRam = avgPresent * float64(v.lastRam)
avgDisk = avgPresent * v.lastDisk
avgVcpu = float64(v.sumVcpu) / total
avgRam = float64(v.sumRam) / total
avgDisk = v.sumDisk / total
}
if v.samples > 0 {
tinPct = float64(v.tinHits) * 100 / float64(v.samples)

25
internal/tasks/monthlyAggregate.go
View File

@@ -148,18 +148,16 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
return err
}
if applied, err := db.ApplyLifecycleCreationToSummary(ctx, dbConn, monthlyTable); err != nil {
c.Logger.Warn("failed to apply lifecycle creation times to monthly summary", "error", err, "table", monthlyTable)
} else {
c.Logger.Info("Monthly aggregation creation times", "source_lifecycle_cache", applied)
}
if applied, err := db.ApplyLifecycleDeletionToSummary(ctx, dbConn, monthlyTable, monthStart.Unix(), monthEnd.Unix()); err != nil {
c.Logger.Warn("failed to apply lifecycle deletions to monthly summary", "error", err, "table", monthlyTable)
} else {
c.Logger.Info("Monthly aggregation deletion times", "source_lifecycle_cache", applied)
}
// Backfill missing creation times to the start of the month for rows lacking creation info.
if _, err := dbConn.ExecContext(ctx,
`UPDATE `+monthlyTable+` SET "CreationTime" = $1 WHERE "CreationTime" IS NULL OR "CreationTime" = 0`,
monthStart.Unix(),
); err != nil {
c.Logger.Warn("failed to normalize creation times for monthly summary", "error", err, "table", monthlyTable)
}
rowCount, err := db.TableRowCount(ctx, dbConn, monthlyTable)
if err != nil {
c.Logger.Warn("unable to count monthly summary rows", "error", err, "table", monthlyTable)
@@ -225,6 +223,11 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo
return err
}
if applied, err := db.ApplyLifecycleCreationToSummary(ctx, dbConn, summaryTable); err != nil {
c.Logger.Warn("failed to apply lifecycle creation times to monthly summary (Go)", "error", err, "table", summaryTable)
} else {
c.Logger.Info("Monthly aggregation creation times", "source_lifecycle_cache", applied)
}
if applied, err := db.ApplyLifecycleDeletionToSummary(ctx, dbConn, summaryTable, monthStart.Unix(), monthEnd.Unix()); err != nil {
c.Logger.Warn("failed to apply lifecycle deletions to monthly summary (Go)", "error", err, "table", summaryTable)
} else {
@@ -235,14 +238,6 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo
c.Logger.Warn("failed to refine creation/deletion times (monthly Go)", "error", err, "table", summaryTable)
}
// Backfill missing creation times to the start of the month for rows lacking creation info.
if _, err := dbConn.ExecContext(ctx,
`UPDATE `+summaryTable+` SET "CreationTime" = $1 WHERE "CreationTime" IS NULL OR "CreationTime" = 0`,
monthStart.Unix(),
); err != nil {
c.Logger.Warn("failed to normalize creation times for monthly summary (Go)", "error", err, "table", summaryTable)
}
db.AnalyzeTableIfPostgres(ctx, dbConn, summaryTable)
rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
if err != nil {