diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index ddbd13c..a543c02 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -21,6 +21,14 @@ type SnapshotRecord struct { SnapshotType string } +type SnapshotMigrationStats struct { + HourlyRenamed int + HourlyRegistered int + DailyRegistered int + MonthlyRegistered int + Errors int +} + func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) { dbConn := database.DB() driver := strings.ToLower(dbConn.DriverName()) @@ -95,6 +103,114 @@ CREATE TABLE IF NOT EXISTS snapshot_registry ( } } +func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (SnapshotMigrationStats, error) { + stats := SnapshotMigrationStats{} + if err := EnsureSnapshotRegistry(ctx, database); err != nil { + return stats, err + } + + dbConn := database.DB() + if _, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry`); err != nil { + return stats, fmt.Errorf("unable to clear snapshot registry: %w", err) + } + + allTables, err := ListTablesByPrefix(ctx, database, "inventory_") + if err != nil { + return stats, err + } + tableSet := make(map[string]struct{}, len(allTables)) + for _, table := range allTables { + tableSet[table] = struct{}{} + } + + hourlyTables, err := ListTablesByPrefix(ctx, database, "inventory_hourly_") + if err != nil { + return stats, err + } + for _, table := range hourlyTables { + snapshotTime, err := latestSnapshotTime(ctx, dbConn, table) + if err != nil { + stats.Errors++ + continue + } + if snapshotTime.IsZero() { + suffix := strings.TrimPrefix(table, "inventory_hourly_") + if parsed, parseErr := time.Parse("2006010215", suffix); parseErr == nil { + snapshotTime = parsed + } else if epoch, parseErr := strconv.ParseInt(suffix, 10, 64); parseErr == nil { + snapshotTime = time.Unix(epoch, 0) + } + } + if snapshotTime.IsZero() { + stats.Errors++ + continue + } + + newName := fmt.Sprintf("inventory_hourly_%d", snapshotTime.Unix()) + if newName != table { + if _, exists := tableSet[newName]; exists { + stats.Errors++ + continue + } + if err := renameTable(ctx, dbConn, table, newName); err != nil { + stats.Errors++ + continue + } + delete(tableSet, table) + tableSet[newName] = struct{}{} + table = newName + stats.HourlyRenamed++ + } + + if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil { + stats.Errors++ + continue + } + stats.HourlyRegistered++ + } + + dailyTables, err := ListTablesByPrefix(ctx, database, "inventory_daily_summary_") + if err != nil { + return stats, err + } + for _, table := range dailyTables { + suffix := strings.TrimPrefix(table, "inventory_daily_summary_") + parsed, err := time.Parse("20060102", suffix) + if err != nil { + stats.Errors++ + continue + } + if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil { + stats.Errors++ + continue + } + stats.DailyRegistered++ + } + + monthlyTables, err := ListTablesByPrefix(ctx, database, "inventory_monthly_summary_") + if err != nil { + return stats, err + } + for _, table := range monthlyTables { + suffix := strings.TrimPrefix(table, "inventory_monthly_summary_") + parsed, err := time.Parse("200601", suffix) + if err != nil { + stats.Errors++ + continue + } + if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil { + stats.Errors++ + continue + } + stats.MonthlyRegistered++ + } + + if stats.Errors > 0 { + return stats, fmt.Errorf("migration completed with %d error(s)", stats.Errors) + } + return stats, nil +} + func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error { if snapshotType == "" || tableName == "" { return fmt.Errorf("snapshot type or table name is empty") @@ -248,7 +364,7 @@ ORDER BY snapshot_time ASC, table_name ASC func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName string) string { switch snapshotType { case "hourly": - return snapshotTime.Format("2006-01-02 15:00") + return snapshotTime.Format("2006-01-02 15:04") case "daily": return snapshotTime.Format("2006-01-02") case "monthly": @@ -524,3 +640,32 @@ func formatEpochHuman(value interface{}) string { } return time.Unix(epoch, 0).Local().Format("Mon 02 Jan 2006 15:04:05 MST") } + +func renameTable(ctx context.Context, dbConn *sqlx.DB, oldName string, newName string) error { + if err := validateTableName(oldName); err != nil { + return err + } + if err := validateTableName(newName); err != nil { + return err + } + _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s RENAME TO %s`, oldName, newName)) + if err != nil { + return fmt.Errorf("failed to rename table %s to %s: %w", oldName, newName, err) + } + return nil +} + +func latestSnapshotTime(ctx context.Context, dbConn *sqlx.DB, tableName string) (time.Time, error) { + if err := validateTableName(tableName); err != nil { + return time.Time{}, err + } + query := fmt.Sprintf(`SELECT MAX("SnapshotTime") FROM %s`, tableName) + var maxTime sql.NullInt64 + if err := dbConn.GetContext(ctx, &maxTime, query); err != nil { + return time.Time{}, err + } + if !maxTime.Valid || maxTime.Int64 <= 0 { + return time.Time{}, nil + } + return time.Unix(maxTime.Int64, 0), nil +} diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index c2db192..e55a31d 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -46,6 +46,10 @@ type inventorySnapshotRow struct { // RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table. func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Hourly snapshot job finished", "duration", time.Since(startedAt)) + }() startTime := time.Now() tableName, err := hourlyInventoryTableName(startTime) if err != nil { @@ -101,6 +105,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo // RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table. func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Daily summary job finished", "duration", time.Since(startedAt)) + }() targetTime := time.Now().Add(-time.Minute) return c.aggregateDailySummary(ctx, targetTime, false) } @@ -250,6 +258,10 @@ GROUP BY // RunVcenterMonthlyAggregate summarizes the previous month's daily snapshots. func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Monthly summary job finished", "duration", time.Since(startedAt)) + }() now := time.Now() firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) targetMonth := firstOfThisMonth.AddDate(0, -1, 0) @@ -375,6 +387,10 @@ GROUP BY // RunSnapshotCleanup drops hourly and daily snapshot tables older than retention. func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Snapshot cleanup job finished", "duration", time.Since(startedAt)) + }() now := time.Now() hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60) dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12) diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index 1710458..70b4cda 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -20,6 +20,10 @@ import ( // use gocron to check vcenters for VMs or updates we don't know about func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Vcenter poll job finished", "duration", time.Since(startedAt)) + }() var matchFound bool // reload settings in case vcenter list has changed diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index 377e6b8..4a29dce 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -14,6 +14,10 @@ import ( // use gocron to check events in the Events table func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { + startedAt := time.Now() + defer func() { + logger.Info("Event processing job finished", "duration", time.Since(startedAt)) + }() var ( numVcpus int32 numRam int32 diff --git a/main.go b/main.go index a6ff273..ff9e45e 100644 --- a/main.go +++ b/main.go @@ -167,17 +167,19 @@ func main() { VcCreds: &creds, } - cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60) - logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency) + /* + cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60) + logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency) - cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200) - logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency) + cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200) + logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency) + */ cronSnapshotFrequency = durationFromSeconds(s.Values.Settings.VcenterInventorySnapshotSeconds, 3600) logger.Debug("Setting VM inventory snapshot cronjob frequency to", "frequency", cronSnapshotFrequency) cronAggregateFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryAggregateSeconds, 86400) - logger.Debug("Setting VM inventory aggregation cronjob frequency to", "frequency", cronAggregateFrequency) + logger.Debug("Setting VM inventory daily aggregation cronjob frequency to", "frequency", cronAggregateFrequency) /* // start background processing for events stored in events table @@ -248,8 +250,10 @@ func main() { } logger.Debug("Created vcenter inventory aggregation cron job", "job", job4.ID(), "starting_at", startsAt4) + monthlyCron := "0 0 1 * *" + logger.Debug("Setting monthly aggregation cron schedule", "cron", monthlyCron) job5, err := c.NewJob( - gocron.CronJob("0 0 1 * *", false), + gocron.CronJob(monthlyCron, false), gocron.NewTask(func() { ct.RunVcenterMonthlyAggregate(ctx, logger) }), gocron.WithSingletonMode(gocron.LimitModeReschedule), diff --git a/server/handler/snapshotMigrate.go b/server/handler/snapshotMigrate.go new file mode 100644 index 0000000..36a118d --- /dev/null +++ b/server/handler/snapshotMigrate.go @@ -0,0 +1,38 @@ +package handler + +import ( + "context" + "encoding/json" + "net/http" + "vctp/internal/report" +) + +// SnapshotMigrate rebuilds the snapshot registry and normalizes hourly table names. +// @Summary Migrate snapshot registry +// @Description Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names. +// @Tags snapshots +// @Produce json +// @Success 200 {object} map[string]interface{} "Migration results" +// @Failure 500 {object} map[string]string "Server error" +// @Router /api/snapshots/migrate [post] +func (h *Handler) SnapshotMigrate(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + stats, err := report.MigrateSnapshotRegistry(ctx, h.Database) + if err != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "ERROR", + "error": err.Error(), + "stats": stats, + }) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]interface{}{ + "status": "OK", + "stats": stats, + }) +} diff --git a/server/middleware/logging.go b/server/middleware/logging.go index e50396c..8d71e13 100644 --- a/server/middleware/logging.go +++ b/server/middleware/logging.go @@ -25,10 +25,15 @@ func (l *LoggingMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := time.Now() l.handler.ServeHTTP(w, r) + query := r.URL.RawQuery + if query == "" { + query = "-" + } l.logger.Debug( "Request recieved", slog.String("method", r.Method), slog.String("path", r.URL.Path), + slog.String("query", query), slog.String("remote", r.RemoteAddr), slog.Duration("duration", time.Since(start)), ) diff --git a/server/router/docs/docs.go b/server/router/docs/docs.go index c3ef816..6dcf8fd 100644 --- a/server/router/docs/docs.go +++ b/server/router/docs/docs.go @@ -641,6 +641,36 @@ const docTemplate = `{ } } }, + "/api/snapshots/migrate": { + "post": { + "description": "Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names.", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "Migrate snapshot registry", + "responses": { + "200": { + "description": "Migration results", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/snapshots/daily": { "get": { "description": "Lists daily summary snapshot tables.", diff --git a/server/router/docs/swagger.json b/server/router/docs/swagger.json index 0008260..4cd5b39 100644 --- a/server/router/docs/swagger.json +++ b/server/router/docs/swagger.json @@ -630,6 +630,36 @@ } } }, + "/api/snapshots/migrate": { + "post": { + "description": "Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names.", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "Migrate snapshot registry", + "responses": { + "200": { + "description": "Migration results", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/snapshots/daily": { "get": { "description": "Lists daily summary snapshot tables.", diff --git a/server/router/docs/swagger.yaml b/server/router/docs/swagger.yaml index b467e43..30845d2 100644 --- a/server/router/docs/swagger.yaml +++ b/server/router/docs/swagger.yaml @@ -569,6 +569,27 @@ paths: summary: Force snapshot aggregation tags: - snapshots + /api/snapshots/migrate: + post: + description: Rebuilds the snapshot registry from existing tables and renames + hourly tables to epoch-based names. + produces: + - application/json + responses: + "200": + description: Migration results + schema: + additionalProperties: true + type: object + "500": + description: Server error + schema: + additionalProperties: + type: string + type: object + summary: Migrate snapshot registry + tags: + - snapshots /snapshots/daily: get: description: Lists daily summary snapshot tables. diff --git a/server/router/router.go b/server/router/router.go index dae1177..08bd429 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -52,6 +52,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st mux.HandleFunc("/api/report/updates", h.UpdateReportDownload) mux.HandleFunc("/api/report/snapshot", h.SnapshotReportDownload) mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce) + mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate) mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList) mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList)