nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

potential performance improvements for hourly inventory
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-02-09 17:13:20 +11:00
parent 24bf74ad34
commit e0cbc20140
1 changed files with 33 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/tasks/inventorySnapshots.go
View File

@@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"sort"
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
@@ -164,6 +165,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if concurrencyLimit > 0 { if concurrencyLimit > 0 {
sem = make(chan struct{}, concurrencyLimit) sem = make(chan struct{}, concurrencyLimit)
} }
var deferredReportTables sync.Map
c.Logger.Info("Starting hourly snapshots", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "concurrency_limit", concurrencyLimit) c.Logger.Info("Starting hourly snapshots", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "concurrency_limit", concurrencyLimit)
for _, url := range c.Settings.Values.Settings.VcenterAddresses { for _, url := range c.Settings.Values.Settings.VcenterAddresses {
wg.Add(1) wg.Add(1)
@@ -182,7 +184,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
defer cancel() defer cancel()
c.Logger.Info("Starting hourly snapshot for vcenter", "url", url) c.Logger.Info("Starting hourly snapshot for vcenter", "url", url)
if err := c.captureHourlySnapshotForVcenter(runCtx, startTime, tableName, url); err != nil { if err := c.captureHourlySnapshotForVcenter(runCtx, startTime, tableName, url, &deferredReportTables); err != nil {
atomic.AddInt64(&errCount, 1) atomic.AddInt64(&errCount, 1)
c.Logger.Error("hourly snapshot failed", "error", err, "url", url) c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
} else { } else {
@@ -211,6 +213,22 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
} }
metrics.RecordHourlySnapshot(startTime, rowCount, err) metrics.RecordHourlySnapshot(startTime, rowCount, err)
var deferredTables []string
deferredReportTables.Range(func(key, _ interface{}) bool {
name, ok := key.(string)
if ok && strings.TrimSpace(name) != "" && name != tableName {
deferredTables = append(deferredTables, name)
}
return true
})
sort.Strings(deferredTables)
for _, reportTable := range deferredTables {
if err := c.generateReport(ctx, reportTable); err != nil {
c.Logger.Warn("failed to regenerate deferred hourly report after deletions", "error", err, "table", reportTable)
} else {
c.Logger.Debug("Regenerated deferred hourly report after deletions", "table", reportTable)
}
}
if err := c.generateReport(ctx, tableName); err != nil { if err := c.generateReport(ctx, tableName); err != nil {
c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName) c.Logger.Warn("failed to generate hourly report", "error", err, "table", tableName)
} }
@@ -277,7 +295,7 @@ func (c *CronTask) RunHourlySnapshotRetry(ctx context.Context, logger *slog.Logg
continue continue
} }
logger.Info("Retrying hourly snapshot", "vcenter", f.Vcenter, "snapshot_time", startTime, "attempt", f.Attempts+1) logger.Info("Retrying hourly snapshot", "vcenter", f.Vcenter, "snapshot_time", startTime, "attempt", f.Attempts+1)
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, f.Vcenter); err != nil { if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, f.Vcenter, nil); err != nil {
logger.Warn("retry failed", "vcenter", f.Vcenter, "error", err) logger.Warn("retry failed", "vcenter", f.Vcenter, "error", err)
} }
} }
@@ -1037,7 +1055,7 @@ func (c *CronTask) initVcenterResources(ctx context.Context, log *slog.Logger, u
return vc, res, cleanup, nil return vc, res, cleanup, nil
} }
func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string) error { func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTime time.Time, tableName string, url string, deferredReportTables *sync.Map) error {
log := c.Logger.With("vcenter", url) log := c.Logger.With("vcenter", url)
ctx = context.WithValue(ctx, ctxLoggerKey{}, log) ctx = context.WithValue(ctx, ctxLoggerKey{}, log)
started := time.Now() started := time.Now()
@@ -1356,11 +1374,18 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
if len(reportTables) == 0 { if len(reportTables) == 0 {
reportTables[tableName] = struct{}{} reportTables[tableName] = struct{}{}
} }
for reportTable := range reportTables { if deferredReportTables != nil {
if err := c.generateReport(ctx, reportTable); err != nil { for reportTable := range reportTables {
log.Warn("failed to regenerate hourly report after deletions", "error", err, "table", reportTable) deferredReportTables.Store(reportTable, struct{}{})
} else { }
log.Debug("Regenerated hourly report after deletions", "table", reportTable) log.Debug("Queued hourly report regeneration after deletions", "tables", len(reportTables))
} else {
for reportTable := range reportTables {
if err := c.generateReport(ctx, reportTable); err != nil {
log.Warn("failed to regenerate hourly report after deletions", "error", err, "table", reportTable)
} else {
log.Debug("Regenerated hourly report after deletions", "table", reportTable)
}
} }
} }
} }