nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

postgres optimisations and daily sqlite vacuum
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-15 19:43:20 +11:00
parent 75a5f31a2f
commit f0bacab729
6 changed files with 56 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/report/snapshots.go
View File

@@ -99,6 +99,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
return err
}
_, _ = dbConn.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_snapshot_registry_type_time ON snapshot_registry (snapshot_type, snapshot_time)`)
_, _ = dbConn.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_snapshot_registry_table_name ON snapshot_registry (table_name)`)
return nil
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
@@ -117,6 +119,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
return err
}
_, _ = dbConn.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_snapshot_registry_type_time ON snapshot_registry (snapshot_type, snapshot_time DESC)`)
_, _ = dbConn.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_snapshot_registry_table_name ON snapshot_registry (table_name)`)
return nil
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)

1
internal/settings/settings.go
View File

@@ -52,6 +52,7 @@ type SettingsYML struct {
NodeChargeClusters []string `yaml:"node_charge_clusters"`
SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"`
VcenterAddresses []string `yaml:"vcenter_addresses"`
PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"`
} `yaml:"settings"`
}

2
internal/tasks/dailyAggregate.go
View File

@@ -37,6 +37,7 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
}
dbConn := c.Database.DB()
db.SetPostgresWorkMem(ctx, dbConn, c.Settings.Values.Settings.PostgresWorkMemMB)
if err := db.EnsureSummaryTable(ctx, dbConn, summaryTable); err != nil {
return err
}
@@ -139,6 +140,7 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
if err := db.RefineCreationDeletionFromUnion(ctx, dbConn, summaryTable, unionQuery); err != nil {
c.Logger.Warn("failed to refine creation/deletion times", "error", err, "table", summaryTable)
}
db.AnalyzeTableIfPostgres(ctx, dbConn, summaryTable)
rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
if err != nil {
c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable)

3
internal/tasks/monthlyAggregate.go
View File

@@ -44,6 +44,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
dailySnapshots = filterRecordsInRange(dailySnapshots, monthStart, monthEnd)
dbConn := c.Database.DB()
db.SetPostgresWorkMem(ctx, dbConn, c.Settings.Values.Settings.PostgresWorkMemMB)
dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots)
if len(dailySnapshots) == 0 {
return fmt.Errorf("no hourly snapshot tables found for %s", targetMonth.Format("2006-01"))
@@ -114,6 +115,8 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
}
db.AnalyzeTableIfPostgres(ctx, dbConn, monthlyTable)
if err := c.generateReport(ctx, monthlyTable); err != nil {
c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable)
metrics.RecordMonthlyAggregation(time.Since(jobStart), err)