nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adjustments to reporting
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-14 10:23:25 +11:00
parent 7b600b2359
commit b297b8293c
11 changed files with 305 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
internal/report/snapshots.go
View File

@@ -21,6 +21,14 @@ type SnapshotRecord struct {
SnapshotType string
}
type SnapshotMigrationStats struct {
HourlyRenamed int
HourlyRegistered int
DailyRegistered int
MonthlyRegistered int
Errors int
}
func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) {
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
@@ -95,6 +103,114 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
}
}
func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (SnapshotMigrationStats, error) {
stats := SnapshotMigrationStats{}
if err := EnsureSnapshotRegistry(ctx, database); err != nil {
return stats, err
}
dbConn := database.DB()
if _, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry`); err != nil {
return stats, fmt.Errorf("unable to clear snapshot registry: %w", err)
}
allTables, err := ListTablesByPrefix(ctx, database, "inventory_")
if err != nil {
return stats, err
}
tableSet := make(map[string]struct{}, len(allTables))
for _, table := range allTables {
tableSet[table] = struct{}{}
}
hourlyTables, err := ListTablesByPrefix(ctx, database, "inventory_hourly_")
if err != nil {
return stats, err
}
for _, table := range hourlyTables {
snapshotTime, err := latestSnapshotTime(ctx, dbConn, table)
if err != nil {
stats.Errors++
continue
}
if snapshotTime.IsZero() {
suffix := strings.TrimPrefix(table, "inventory_hourly_")
if parsed, parseErr := time.Parse("2006010215", suffix); parseErr == nil {
snapshotTime = parsed
} else if epoch, parseErr := strconv.ParseInt(suffix, 10, 64); parseErr == nil {
snapshotTime = time.Unix(epoch, 0)
}
}
if snapshotTime.IsZero() {
stats.Errors++
continue
}
newName := fmt.Sprintf("inventory_hourly_%d", snapshotTime.Unix())
if newName != table {
if _, exists := tableSet[newName]; exists {
stats.Errors++
continue
}
if err := renameTable(ctx, dbConn, table, newName); err != nil {
stats.Errors++
continue
}
delete(tableSet, table)
tableSet[newName] = struct{}{}
table = newName
stats.HourlyRenamed++
}
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
stats.Errors++
continue
}
stats.HourlyRegistered++
}
dailyTables, err := ListTablesByPrefix(ctx, database, "inventory_daily_summary_")
if err != nil {
return stats, err
}
for _, table := range dailyTables {
suffix := strings.TrimPrefix(table, "inventory_daily_summary_")
parsed, err := time.Parse("20060102", suffix)
if err != nil {
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
stats.Errors++
continue
}
stats.DailyRegistered++
}
monthlyTables, err := ListTablesByPrefix(ctx, database, "inventory_monthly_summary_")
if err != nil {
return stats, err
}
for _, table := range monthlyTables {
suffix := strings.TrimPrefix(table, "inventory_monthly_summary_")
parsed, err := time.Parse("200601", suffix)
if err != nil {
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
stats.Errors++
continue
}
stats.MonthlyRegistered++
}
if stats.Errors > 0 {
return stats, fmt.Errorf("migration completed with %d error(s)", stats.Errors)
}
return stats, nil
}
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty")
@@ -248,7 +364,7 @@ ORDER BY snapshot_time ASC, table_name ASC
func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName string) string {
switch snapshotType {
case "hourly":
return snapshotTime.Format("2006-01-02 15:00")
return snapshotTime.Format("2006-01-02 15:04")
case "daily":
return snapshotTime.Format("2006-01-02")
case "monthly":
@@ -524,3 +640,32 @@ func formatEpochHuman(value interface{}) string {
}
return time.Unix(epoch, 0).Local().Format("Mon 02 Jan 2006 15:04:05 MST")
}
func renameTable(ctx context.Context, dbConn *sqlx.DB, oldName string, newName string) error {
if err := validateTableName(oldName); err != nil {
return err
}
if err := validateTableName(newName); err != nil {
return err
}
_, err := dbConn.ExecContext(ctx, fmt.Sprintf(`ALTER TABLE %s RENAME TO %s`, oldName, newName))
if err != nil {
return fmt.Errorf("failed to rename table %s to %s: %w", oldName, newName, err)
}
return nil
}
func latestSnapshotTime(ctx context.Context, dbConn *sqlx.DB, tableName string) (time.Time, error) {
if err := validateTableName(tableName); err != nil {
return time.Time{}, err
}
query := fmt.Sprintf(`SELECT MAX("SnapshotTime") FROM %s`, tableName)
var maxTime sql.NullInt64
if err := dbConn.GetContext(ctx, &maxTime, query); err != nil {
return time.Time{}, err
}
if !maxTime.Valid || maxTime.Int64 <= 0 {
return time.Time{}, nil
}
return time.Unix(maxTime.Int64, 0), nil
}

16
internal/tasks/inventorySnapshots.go
View File

@@ -46,6 +46,10 @@ type inventorySnapshotRow struct {
// RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table.
func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Hourly snapshot job finished", "duration", time.Since(startedAt))
}()
startTime := time.Now()
tableName, err := hourlyInventoryTableName(startTime)
if err != nil {
@@ -101,6 +105,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
// RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table.
func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Daily summary job finished", "duration", time.Since(startedAt))
}()
targetTime := time.Now().Add(-time.Minute)
return c.aggregateDailySummary(ctx, targetTime, false)
}
@@ -250,6 +258,10 @@ GROUP BY
// RunVcenterMonthlyAggregate summarizes the previous month's daily snapshots.
func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Monthly summary job finished", "duration", time.Since(startedAt))
}()
now := time.Now()
firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
targetMonth := firstOfThisMonth.AddDate(0, -1, 0)
@@ -375,6 +387,10 @@ GROUP BY
// RunSnapshotCleanup drops hourly and daily snapshot tables older than retention.
func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Snapshot cleanup job finished", "duration", time.Since(startedAt))
}()
now := time.Now()
hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60)
dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12)

4
internal/tasks/monitorVcenter.go
View File

@@ -20,6 +20,10 @@ import (
// use gocron to check vcenters for VMs or updates we don't know about
func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Vcenter poll job finished", "duration", time.Since(startedAt))
}()
var matchFound bool
// reload settings in case vcenter list has changed

4
internal/tasks/processEvents.go
View File

@@ -14,6 +14,10 @@ import (
// use gocron to check events in the Events table
func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
startedAt := time.Now()
defer func() {
logger.Info("Event processing job finished", "duration", time.Since(startedAt))
}()
var (
numVcpus int32
numRam int32

8
main.go
View File

@@ -167,17 +167,19 @@ func main() {
VcCreds: &creds,
}
/*
cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60)
logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency)
cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200)
logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency)
*/
cronSnapshotFrequency = durationFromSeconds(s.Values.Settings.VcenterInventorySnapshotSeconds, 3600)
logger.Debug("Setting VM inventory snapshot cronjob frequency to", "frequency", cronSnapshotFrequency)
cronAggregateFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryAggregateSeconds, 86400)
logger.Debug("Setting VM inventory aggregation cronjob frequency to", "frequency", cronAggregateFrequency)
logger.Debug("Setting VM inventory daily aggregation cronjob frequency to", "frequency", cronAggregateFrequency)
/*
// start background processing for events stored in events table
@@ -248,8 +250,10 @@ func main() {
}
logger.Debug("Created vcenter inventory aggregation cron job", "job", job4.ID(), "starting_at", startsAt4)
monthlyCron := "0 0 1 * *"
logger.Debug("Setting monthly aggregation cron schedule", "cron", monthlyCron)
job5, err := c.NewJob(
gocron.CronJob("0 0 1 * *", false),
gocron.CronJob(monthlyCron, false),
gocron.NewTask(func() {
ct.RunVcenterMonthlyAggregate(ctx, logger)
}), gocron.WithSingletonMode(gocron.LimitModeReschedule),

38
server/handler/snapshotMigrate.go Normal file
View File

@@ -0,0 +1,38 @@
package handler
import (
"context"
"encoding/json"
"net/http"
"vctp/internal/report"
)
// SnapshotMigrate rebuilds the snapshot registry and normalizes hourly table names.
// @Summary Migrate snapshot registry
// @Description Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names.
// @Tags snapshots
// @Produce json
// @Success 200 {object} map[string]interface{} "Migration results"
// @Failure 500 {object} map[string]string "Server error"
// @Router /api/snapshots/migrate [post]
func (h *Handler) SnapshotMigrate(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
stats, err := report.MigrateSnapshotRegistry(ctx, h.Database)
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusInternalServerError)
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "ERROR",
"error": err.Error(),
"stats": stats,
})
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]interface{}{
"status": "OK",
"stats": stats,
})
}

5
server/middleware/logging.go
View File

@@ -25,10 +25,15 @@ func (l *LoggingMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
start := time.Now()
l.handler.ServeHTTP(w, r)
query := r.URL.RawQuery
if query == "" {
query = "-"
}
l.logger.Debug(
"Request recieved",
slog.String("method", r.Method),
slog.String("path", r.URL.Path),
slog.String("query", query),
slog.String("remote", r.RemoteAddr),
slog.Duration("duration", time.Since(start)),
)

30
server/router/docs/docs.go
View File

@@ -641,6 +641,36 @@ const docTemplate = `{
}
}
},
"/api/snapshots/migrate": {
"post": {
"description": "Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Migrate snapshot registry",
"responses": {
"200": {
"description": "Migration results",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/snapshots/daily": {
"get": {
"description": "Lists daily summary snapshot tables.",

30
server/router/docs/swagger.json
View File

@@ -630,6 +630,36 @@
}
}
},
"/api/snapshots/migrate": {
"post": {
"description": "Rebuilds the snapshot registry from existing tables and renames hourly tables to epoch-based names.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Migrate snapshot registry",
"responses": {
"200": {
"description": "Migration results",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/snapshots/daily": {
"get": {
"description": "Lists daily summary snapshot tables.",

21
server/router/docs/swagger.yaml
View File

@@ -569,6 +569,27 @@ paths:
summary: Force snapshot aggregation
tags:
- snapshots
/api/snapshots/migrate:
post:
description: Rebuilds the snapshot registry from existing tables and renames
hourly tables to epoch-based names.
produces:
- application/json
responses:
"200":
description: Migration results
schema:
additionalProperties: true
type: object
"500":
description: Server error
schema:
additionalProperties:
type: string
type: object
summary: Migrate snapshot registry
tags:
- snapshots
/snapshots/daily:
get:
description: Lists daily summary snapshot tables.

1
server/router/router.go
View File

@@ -52,6 +52,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
mux.HandleFunc("/api/report/updates", h.UpdateReportDownload)
mux.HandleFunc("/api/report/snapshot", h.SnapshotReportDownload)
mux.HandleFunc("/api/snapshots/aggregate", h.SnapshotAggregateForce)
mux.HandleFunc("/api/snapshots/migrate", h.SnapshotMigrate)
mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList)
mux.HandleFunc("/snapshots/daily", h.SnapshotDailyList)