From e5e5be37a36e5f7a222449504da982e6db574d75 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Thu, 15 Jan 2026 16:02:58 +1100 Subject: [PATCH] handle crashes better --- internal/tasks/cronstatus.go | 9 +++++++++ internal/tasks/inventorySnapshots.go | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/tasks/cronstatus.go b/internal/tasks/cronstatus.go index 23a0c8a..cbd0c6f 100644 --- a/internal/tasks/cronstatus.go +++ b/internal/tasks/cronstatus.go @@ -21,6 +21,15 @@ func NewCronTracker(database db.Database) *CronTracker { } } +// ClearAllInProgress resets any stuck in-progress flags (e.g., after crashes). +func (c *CronTracker) ClearAllInProgress(ctx context.Context) error { + if err := c.ensureTable(ctx); err != nil { + return err + } + _, err := c.db.DB().ExecContext(ctx, `UPDATE cron_status SET in_progress = FALSE`) + return err +} + func (c *CronTracker) ensureTable(ctx context.Context) error { conn := c.db.DB() driver := conn.DriverName() diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index 71de41b..e6eaacd 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -57,11 +57,16 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo jobCtx, cancel = context.WithTimeout(ctx, jobTimeout) defer cancel() } + tracker := NewCronTracker(c.Database) + // Clear any stale in-progress markers (e.g., after a crash) before attempting the run. + if err := tracker.ClearAllInProgress(jobCtx); err != nil { + logger.Warn("failed to clear stale cron status", "error", err) + } + startedAt := time.Now() defer func() { logger.Info("Hourly snapshot job finished", "duration", time.Since(startedAt)) }() - tracker := NewCronTracker(c.Database) done, skip, err := tracker.Start(jobCtx, "hourly_snapshot") if err != nil { return err