adjustments to reporting
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-14 10:23:25 +11:00
parent 7b600b2359
commit b297b8293c
11 changed files with 305 additions and 7 deletions

View File

@@ -21,6 +21,14 @@ type SnapshotRecord struct {
SnapshotType string
}
type SnapshotMigrationStats struct {
HourlyRenamed int
HourlyRegistered int
DailyRegistered int
MonthlyRegistered int
Errors int
}
func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) {
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
@@ -95,6 +103,114 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
}
}
func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (SnapshotMigrationStats, error) {
stats := SnapshotMigrationStats{}
if err := EnsureSnapshotRegistry(ctx, database); err != nil {
return stats, err
}
dbConn := database.DB()
if _, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry`); err != nil {
return stats, fmt.Errorf("unable to clear snapshot registry: %w", err)
}
allTables, err := ListTablesByPrefix(ctx, database, "inventory_")
if err != nil {
return stats, err
}
tableSet := make(map[string]struct{}, len(allTables))
for _, table := range allTables {
tableSet[table] = struct{}{}
}
hourlyTables, err := ListTablesByPrefix(ctx, database, "inventory_hourly_")
if err != nil {
return stats, err
}
for _, table := range hourlyTables {
snapshotTime, err := latestSnapshotTime(ctx, dbConn, table)
if err != nil {
stats.Errors++
continue
}
if snapshotTime.IsZero() {
suffix := strings.TrimPrefix(table, "inventory_hourly_")
if parsed, parseErr := time.Parse("2006010215", suffix); parseErr == nil {
snapshotTime = parsed
} else if epoch, parseErr := strconv.ParseInt(suffix, 10, 64); parseErr == nil {
snapshotTime = time.Unix(epoch, 0)
}
}
if snapshotTime.IsZero() {
stats.Errors++
continue
}
newName := fmt.Sprintf("inventory_hourly_%d", snapshotTime.Unix())
if newName != table {
if _, exists := tableSet[newName]; exists {
stats.Errors++
continue
}
if err := renameTable(ctx, dbConn, table, newName); err != nil {
stats.Errors++
continue
}
delete(tableSet, table)
tableSet[newName] = struct{}{}
table = newName
stats.HourlyRenamed++
}
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
stats.Errors++
continue
}
stats.HourlyRegistered++
}
dailyTables, err := ListTablesByPrefix(ctx, database, "inventory_daily_summary_")
if err != nil {
return stats, err
}
for _, table := range dailyTables {
suffix := strings.TrimPrefix(table, "inventory_daily_summary_")
parsed, err := time.Parse("20060102", suffix)
if err != nil {
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
stats.Errors++
continue
}
stats.DailyRegistered++
}
monthlyTables, err := ListTablesByPrefix(ctx, database, "inventory_monthly_summary_")
if err != nil {
return stats, err
}
for _, table := range monthlyTables {
suffix := strings.TrimPrefix(table, "inventory_monthly_summary_")
parsed, err := time.Parse("200601", suffix)
if err != nil {
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
stats.Errors++
continue
}
stats.MonthlyRegistered++
}
if stats.Errors > 0 {
return stats, fmt.Errorf("migration completed with %d error(s)", stats.Errors)
}
return stats, nil
}
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty")
@@ -248,7 +364,7 @@ ORDER BY snapshot_time ASC, table_name ASC
func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName string) string {
switch snapshotType {
case "hourly":
return snapshotTime.Format("2006-01-02 15:00")
return snapshotTime.Format("2006-01-02 15:04")
case "daily":
return snapshotTime.Format("2006-01-02")
case "monthly":
@@ -524,3 +640,32 @@ func formatEpochHuman(value interface{}) string {
}
return time.Unix(epoch, 0).Local().Format("Mon 02 Jan 2006 15:04:05 MST")
}
func renameTable(ctx context.Context, dbConn *sqlx.DB, oldName string, newName string) error {
if err := validateTableName(oldName); err != nil {
return err
}
if err := validateTableName(newName); err != nil {
return err
}
_, err := dbConn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s RENAME TO %s`, oldName, newName))
if err != nil {
return fmt.Errorf("failed to rename table %s to %s: %w", oldName, newName, err)
}
return nil
}
func latestSnapshotTime(ctx context.Context, dbConn *sqlx.DB, tableName string) (time.Time, error) {
if err := validateTableName(tableName); err != nil {
return time.Time{}, err
}
query := fmt.Sprintf(`SELECT MAX("SnapshotTime") FROM %s`, tableName)
var maxTime sql.NullInt64
if err := dbConn.GetContext(ctx, &maxTime, query); err != nil {
return time.Time{}, err
}
if !maxTime.Valid || maxTime.Int64 <= 0 {
return time.Time{}, nil
}
return time.Unix(maxTime.Int64, 0), nil
}

View File

@@ -46,6 +46,10 @@ type inventorySnapshotRow struct {
// RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table.
func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Hourly snapshot job finished", "duration", time.Since(startedAt))
}()
startTime := time.Now()
tableName, err := hourlyInventoryTableName(startTime)
if err != nil {
@@ -101,6 +105,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
// RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table.
func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Daily summary job finished", "duration", time.Since(startedAt))
}()
targetTime := time.Now().Add(-time.Minute)
return c.aggregateDailySummary(ctx, targetTime, false)
}
@@ -250,6 +258,10 @@ GROUP BY
// RunVcenterMonthlyAggregate summarizes the previous month's daily snapshots.
func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Monthly summary job finished", "duration", time.Since(startedAt))
}()
now := time.Now()
firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
targetMonth := firstOfThisMonth.AddDate(0, -1, 0)
@@ -375,6 +387,10 @@ GROUP BY
// RunSnapshotCleanup drops hourly and daily snapshot tables older than retention.
func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Snapshot cleanup job finished", "duration", time.Since(startedAt))
}()
now := time.Now()
hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60)
dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12)

View File

@@ -20,6 +20,10 @@ import (
// use gocron to check vcenters for VMs or updates we don't know about
func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Vcenter poll job finished", "duration", time.Since(startedAt))
}()
var matchFound bool
// reload settings in case vcenter list has changed

View File

@@ -14,6 +14,10 @@ import (
// use gocron to check events in the Events table
func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Event processing job finished", "duration", time.Since(startedAt))
}()
var (
numVcpus int32
numRam int32