handle crashes better
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-15 16:02:58 +11:00
parent 96567f6211
commit e5e5be37a3
2 changed files with 15 additions and 1 deletions

View File

@@ -21,6 +21,15 @@ func NewCronTracker(database db.Database) *CronTracker {
}
}
// ClearAllInProgress resets any stuck in-progress flags (e.g., after crashes).
func (c *CronTracker) ClearAllInProgress(ctx context.Context) error {
if err := c.ensureTable(ctx); err != nil {
return err
}
_, err := c.db.DB().ExecContext(ctx, `UPDATE cron_status SET in_progress = FALSE`)
return err
}
func (c *CronTracker) ensureTable(ctx context.Context) error {
conn := c.db.DB()
driver := conn.DriverName()

View File

@@ -57,11 +57,16 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
jobCtx, cancel = context.WithTimeout(ctx, jobTimeout)
defer cancel()
}
tracker := NewCronTracker(c.Database)
// Clear any stale in-progress markers (e.g., after a crash) before attempting the run.
if err := tracker.ClearAllInProgress(jobCtx); err != nil {
logger.Warn("failed to clear stale cron status", "error", err)
}
startedAt := time.Now()
defer func() {
logger.Info("Hourly snapshot job finished", "duration", time.Since(startedAt))
}()
tracker := NewCronTracker(c.Database)
done, skip, err := tracker.Start(jobCtx, "hourly_snapshot")
if err != nil {
return err