add record size to hourly snapshot page
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-14 19:43:32 +11:00
parent 9be3a3d807
commit 8df1d145f8
9 changed files with 220 additions and 165 deletions

View File

@@ -7,6 +7,7 @@ import (
type SnapshotEntry struct { type SnapshotEntry struct {
Label string Label string
Link string Link string
Count int64
} }
templ SnapshotHourlyList(entries []SnapshotEntry) { templ SnapshotHourlyList(entries []SnapshotEntry) {
@@ -46,7 +47,10 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {
<ul class="mt-6 space-y-3 web2-list"> <ul class="mt-6 space-y-3 web2-list">
for _, entry := range entries { for _, entry := range entries {
<li class="flex items-center justify-between gap-4"> <li class="flex items-center justify-between gap-4">
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span> <div class="flex flex-col">
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
<span class="text-xs text-slate-500">{entry.Count} records</span>
</div>
<a class="web2-link" href={entry.Link}>Download XLSX</a> <a class="web2-link" href={entry.Link}>Download XLSX</a>
</li> </li>
} }

View File

@@ -15,6 +15,7 @@ import (
type SnapshotEntry struct { type SnapshotEntry struct {
Label string Label string
Link string Link string
Count int64
} }
func SnapshotHourlyList(entries []SnapshotEntry) templ.Component { func SnapshotHourlyList(entries []SnapshotEntry) templ.Component {
@@ -140,7 +141,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var5 string var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title) templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49} return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 49}
} }
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
@@ -153,7 +154,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var6 string var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle) templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55} return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 36, Col: 55}
} }
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
@@ -166,7 +167,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var7 string var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries)) templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83} return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 45, Col: 83}
} }
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
@@ -177,38 +178,51 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
for _, entry := range entries { for _, entry := range entries {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><span class=\"text-sm font-semibold text-slate-700\">") templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><div class=\"flex flex-col\"><span class=\"text-sm font-semibold text-slate-700\">")
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
var templ_7745c5c3_Var8 string var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label) templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71} return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 51, Col: 72}
} }
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <a class=\"web2-link\" href=\"") templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <span class=\"text-xs text-slate-500\">")
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
var templ_7745c5c3_Var9 templ.SafeURL var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link) templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Count)
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45} return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 52, Col: 58}
} }
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">Download XLSX</a></li>") templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, " records</span></div><a class=\"web2-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 templ.SafeURL
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 54, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\">Download XLSX</a></li>")
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
} }
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</ul></section></main></body>") templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</ul></section></main></body>")
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
@@ -216,7 +230,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</html>") templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</html>")
if templ_7745c5c3_Err != nil { if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err return templ_7745c5c3_Err
} }

View File

@@ -24,6 +24,48 @@ type ColumnDef struct {
Type string Type string
} }
// TableRowCount returns COUNT(*) for a table.
func TableRowCount(ctx context.Context, dbConn *sqlx.DB, table string) (int64, error) {
if err := ValidateTableName(table); err != nil {
return 0, err
}
var count int64
query := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, table)
if err := dbConn.GetContext(ctx, &count, query); err != nil {
return 0, err
}
return count, nil
}
// EnsureColumns adds the provided columns to a table if they are missing.
func EnsureColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []ColumnDef) error {
if _, err := SafeTableName(tableName); err != nil {
return err
}
for _, column := range columns {
if err := AddColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
return err
}
}
return nil
}
// AddColumnIfMissing performs a best-effort ALTER TABLE to add a column, ignoring "already exists".
func AddColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column ColumnDef) error {
if _, err := SafeTableName(tableName); err != nil {
return err
}
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
errText := strings.ToLower(err.Error())
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
return nil
}
return err
}
return nil
}
// ValidateTableName ensures table identifiers are safe for interpolation. // ValidateTableName ensures table identifiers are safe for interpolation.
func ValidateTableName(name string) error { func ValidateTableName(name string) error {
if name == "" { if name == "" {

View File

@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE snapshot_registry DROP COLUMN snapshot_count;

View File

@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE snapshot_registry ADD COLUMN IF NOT EXISTS snapshot_count BIGINT NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE snapshot_registry DROP COLUMN IF EXISTS snapshot_count;

View File

@@ -70,7 +70,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
"id" INTEGER PRIMARY KEY AUTOINCREMENT, "id" INTEGER PRIMARY KEY AUTOINCREMENT,
"snapshot_type" TEXT NOT NULL, "snapshot_type" TEXT NOT NULL,
"table_name" TEXT NOT NULL UNIQUE, "table_name" TEXT NOT NULL UNIQUE,
"snapshot_time" INTEGER NOT NULL "snapshot_time" INTEGER NOT NULL,
"snapshot_count" BIGINT NOT NULL DEFAULT 0
); );
-- The following tables are declared for sqlc type-checking only. -- The following tables are declared for sqlc type-checking only.

View File

@@ -16,9 +16,10 @@ import (
) )
type SnapshotRecord struct { type SnapshotRecord struct {
TableName string TableName string
SnapshotTime time.Time SnapshotTime time.Time
SnapshotType string SnapshotType string
SnapshotCount int64
} }
type SnapshotMigrationStats struct { type SnapshotMigrationStats struct {
@@ -84,20 +85,36 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_type TEXT NOT NULL, snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE, table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
) )
`) `)
return err if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
return err
}
return nil
case "pgx", "postgres": case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, ` _, err := dbConn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS snapshot_registry ( CREATE TABLE IF NOT EXISTS snapshot_registry (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
snapshot_type TEXT NOT NULL, snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE, table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
) )
`) `)
return err if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
return err
}
return nil
default: default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver) return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
} }
@@ -162,7 +179,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.HourlyRenamed++ stats.HourlyRenamed++
} }
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil { rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime, rowCount); err != nil {
stats.Errors++ stats.Errors++
continue continue
} }
@@ -180,7 +198,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++ stats.Errors++
continue continue
} }
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil { rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "daily", table, parsed, rowCount); err != nil {
stats.Errors++ stats.Errors++
continue continue
} }
@@ -198,7 +217,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++ stats.Errors++
continue continue
} }
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil { rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed, rowCount); err != nil {
stats.Errors++ stats.Errors++
continue continue
} }
@@ -211,7 +231,7 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
return stats, nil return stats, nil
} }
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error { func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time, snapshotCount int64) error {
if snapshotType == "" || tableName == "" { if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty") return fmt.Errorf("snapshot type or table name is empty")
} }
@@ -220,16 +240,23 @@ func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType st
switch driver { switch driver {
case "sqlite": case "sqlite":
_, err := dbConn.ExecContext(ctx, ` _, err := dbConn.ExecContext(ctx, `
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time) INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES (?, ?, ?) VALUES (?, ?, ?, ?)
`, snapshotType, tableName, snapshotTime.Unix()) ON CONFLICT(table_name) DO UPDATE SET
snapshot_time = excluded.snapshot_time,
snapshot_type = excluded.snapshot_type,
snapshot_count = excluded.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err return err
case "pgx", "postgres": case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, ` _, err := dbConn.ExecContext(ctx, `
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES ($1, $2, $3) VALUES ($1, $2, $3, $4)
ON CONFLICT (table_name) DO NOTHING ON CONFLICT (table_name) DO UPDATE SET
`, snapshotType, tableName, snapshotTime.Unix()) snapshot_time = EXCLUDED.snapshot_time,
snapshot_type = EXCLUDED.snapshot_type,
snapshot_count = EXCLUDED.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err return err
default: default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver) return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
@@ -264,14 +291,14 @@ func ListSnapshots(ctx context.Context, database db.Database, snapshotType strin
switch driver { switch driver {
case "sqlite": case "sqlite":
rows, err = dbConn.QueryxContext(ctx, ` rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry FROM snapshot_registry
WHERE snapshot_type = ? WHERE snapshot_type = ?
ORDER BY snapshot_time DESC, table_name DESC ORDER BY snapshot_time DESC, table_name DESC
`, snapshotType) `, snapshotType)
case "pgx", "postgres": case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, ` rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry FROM snapshot_registry
WHERE snapshot_type = $1 WHERE snapshot_type = $1
ORDER BY snapshot_time DESC, table_name DESC ORDER BY snapshot_time DESC, table_name DESC
@@ -291,14 +318,16 @@ ORDER BY snapshot_time DESC, table_name DESC
tableName string tableName string
snapshotTime int64 snapshotTime int64
recordType string recordType string
snapshotCnt int64
) )
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil { if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err return nil, err
} }
records = append(records, SnapshotRecord{ records = append(records, SnapshotRecord{
TableName: tableName, TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0), SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType, SnapshotType: recordType,
SnapshotCount: snapshotCnt,
}) })
} }
return records, rows.Err() return records, rows.Err()
@@ -317,7 +346,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
switch driver { switch driver {
case "sqlite": case "sqlite":
rows, err = dbConn.QueryxContext(ctx, ` rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry FROM snapshot_registry
WHERE snapshot_type = ? WHERE snapshot_type = ?
AND snapshot_time >= ? AND snapshot_time >= ?
@@ -326,7 +355,7 @@ ORDER BY snapshot_time ASC, table_name ASC
`, snapshotType, startUnix, endUnix) `, snapshotType, startUnix, endUnix)
case "pgx", "postgres": case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, ` rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry FROM snapshot_registry
WHERE snapshot_type = $1 WHERE snapshot_type = $1
AND snapshot_time >= $2 AND snapshot_time >= $2
@@ -348,14 +377,16 @@ ORDER BY snapshot_time ASC, table_name ASC
tableName string tableName string
snapshotTime int64 snapshotTime int64
recordType string recordType string
snapshotCnt int64
) )
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil { if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err return nil, err
} }
records = append(records, SnapshotRecord{ records = append(records, SnapshotRecord{
TableName: tableName, TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0), SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType, SnapshotType: recordType,
SnapshotCount: snapshotCnt,
}) })
} }
return records, rows.Err() return records, rows.Err()

View File

@@ -86,9 +86,6 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil { if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
return err return err
} }
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil {
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
}
var wg sync.WaitGroup var wg sync.WaitGroup
var errCount int64 var errCount int64
@@ -102,6 +99,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
wg.Add(1) wg.Add(1)
go func(url string) { go func(url string) {
defer wg.Done() defer wg.Done()
vcStart := time.Now()
if sem != nil { if sem != nil {
sem <- struct{}{} sem <- struct{}{}
defer func() { <-sem }() defer func() { <-sem }()
@@ -110,6 +108,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil { if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil {
atomic.AddInt64(&errCount, 1) atomic.AddInt64(&errCount, 1)
c.Logger.Error("hourly snapshot failed", "error", err, "url", url) c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
} else {
c.Logger.Info("Finished hourly snapshot for vcenter", "url", url, "duration", time.Since(vcStart))
} }
}(url) }(url)
} }
@@ -118,7 +118,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount) return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount)
} }
c.Logger.Debug("Finished hourly vcenter snapshot") rowCount, err := db.TableRowCount(ctx, dbConn, tableName)
if err != nil {
c.Logger.Warn("unable to count hourly snapshot rows", "error", err, "table", tableName)
}
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime, rowCount); err != nil {
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
}
c.Logger.Debug("Finished hourly vcenter snapshot", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "table", tableName, "row_count", rowCount)
return nil return nil
} }
@@ -161,12 +169,6 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
return err return err
} }
} }
if rowsExist, err := db.TableHasRows(ctx, dbConn, summaryTable); err != nil {
return err
} else if rowsExist {
c.Logger.Debug("Daily summary already exists, skipping aggregation", "summary_table", summaryTable)
return nil
}
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd) hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd)
if err != nil { if err != nil {
@@ -181,12 +183,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
for _, snapshot := range hourlySnapshots { for _, snapshot := range hourlySnapshots {
hourlyTables = append(hourlyTables, snapshot.TableName) hourlyTables = append(hourlyTables, snapshot.TableName)
} }
unionQuery := buildUnionQuery(hourlyTables, []string{ unionQuery, err := buildUnionQuery(hourlyTables, summaryUnionColumns, templateExclusionFilter())
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, if err != nil {
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, return err
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, }
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery) currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
if err != nil { if err != nil {
@@ -210,24 +210,23 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
for _, snapshot := range prevSnapshots { for _, snapshot := range prevSnapshots {
prevTables = append(prevTables, snapshot.TableName) prevTables = append(prevTables, snapshot.TableName)
} }
prevUnion := buildUnionQuery(prevTables, []string{ prevUnion, err := buildUnionQuery(prevTables, summaryUnionColumns, templateExclusionFilter())
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, if err == nil {
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, if err != nil {
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`, c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
}, templateExclusionFilter()) } else {
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion) c.Logger.Info("Daily snapshot comparison",
if err != nil { "current_date", dayStart.Format("2006-01-02"),
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02")) "previous_date", prevStart.Format("2006-01-02"),
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
)
}
} else { } else {
c.Logger.Info("Daily snapshot comparison", c.Logger.Warn("unable to build previous day union", "error", err)
"current_date", dayStart.Format("2006-01-02"),
"previous_date", prevStart.Format("2006-01-02"),
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
)
} }
} }
@@ -240,7 +239,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02")) c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
return err return err
} }
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart); err != nil { rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
if err != nil {
c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable)
}
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart, rowCount); err != nil {
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable) c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
} }
@@ -300,25 +303,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
return err return err
} }
} }
if rowsExist, err := db.TableHasRows(ctx, dbConn, monthlyTable); err != nil {
return err
} else if rowsExist {
c.Logger.Debug("Monthly summary already exists, skipping aggregation", "summary_table", monthlyTable)
return nil
}
dailyTables := make([]string, 0, len(dailySnapshots)) dailyTables := make([]string, 0, len(dailySnapshots))
for _, snapshot := range dailySnapshots { for _, snapshot := range dailySnapshots {
dailyTables = append(dailyTables, snapshot.TableName) dailyTables = append(dailyTables, snapshot.TableName)
} }
unionQuery := buildUnionQuery(dailyTables, []string{ unionQuery, err := buildUnionQuery(dailyTables, summaryUnionColumns, templateExclusionFilter())
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, if err != nil {
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, return err
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
if strings.TrimSpace(unionQuery) == "" {
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
} }
monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery) monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
@@ -343,7 +335,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01")) c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
return err return err
} }
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil { rowCount, err := db.TableRowCount(ctx, dbConn, monthlyTable)
if err != nil {
c.Logger.Warn("unable to count monthly summary rows", "error", err, "table", monthlyTable)
}
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth, rowCount); err != nil {
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable) c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
} }
@@ -443,25 +439,36 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
return err return err
} }
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ return db.EnsureColumns(ctx, dbConn, tableName, []db.ColumnDef{
{Name: "VcpuCount", Type: "BIGINT"}, {Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"}, {Name: "RamGB", Type: "BIGINT"},
}) })
} }
func buildUnionQuery(tables []string, columns []string, whereClause string) string { func buildUnionQuery(tables []string, columns []string, whereClause string) (string, error) {
if len(tables) == 0 {
return "", fmt.Errorf("no tables provided for union")
}
if len(columns) == 0 {
return "", fmt.Errorf("no columns provided for union")
}
queries := make([]string, 0, len(tables)) queries := make([]string, 0, len(tables))
columnList := strings.Join(columns, ", ") columnList := strings.Join(columns, ", ")
for _, table := range tables { for _, table := range tables {
if _, err := db.SafeTableName(table); err != nil { safeName, err := db.SafeTableName(table)
continue if err != nil {
return "", err
} }
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table) query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
if whereClause != "" { if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause) query = fmt.Sprintf("%s WHERE %s", query, whereClause)
} }
queries = append(queries, query) queries = append(queries, query)
} }
return strings.Join(queries, "\nUNION ALL\n") if len(queries) == 0 {
return "", fmt.Errorf("no valid tables provided for union")
}
return strings.Join(queries, "\nUNION ALL\n"), nil
} }
func templateExclusionFilter() string { func templateExclusionFilter() string {
@@ -525,70 +532,11 @@ type columnDef struct {
Type string Type string
} }
func summaryMetricColumns() []columnDef { var summaryUnionColumns = []string{
return []columnDef{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
{Name: "InventoryId", Type: "BIGINT"}, `"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
{Name: "Name", Type: "TEXT"}, `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
{Name: "Vcenter", Type: "TEXT"}, `"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
{Name: "VmId", Type: "TEXT"},
{Name: "EventKey", Type: "TEXT"},
{Name: "CloudId", Type: "TEXT"},
{Name: "CreationTime", Type: "BIGINT"},
{Name: "DeletionTime", Type: "BIGINT"},
{Name: "ResourcePool", Type: "TEXT"},
{Name: "Datacenter", Type: "TEXT"},
{Name: "Cluster", Type: "TEXT"},
{Name: "Folder", Type: "TEXT"},
{Name: "ProvisionedDisk", Type: "REAL"},
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
{Name: "IsTemplate", Type: "TEXT"},
{Name: "PoweredOn", Type: "TEXT"},
{Name: "SrmPlaceholder", Type: "TEXT"},
{Name: "VmUuid", Type: "TEXT"},
{Name: "SamplesPresent", Type: "BIGINT"},
}
}
func summaryAvgColumns() []columnDef {
return []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
{Name: "AvgProvisionedDisk", Type: "REAL"},
{Name: "AvgIsPresent", Type: "REAL"},
{Name: "PoolTinPct", Type: "REAL"},
{Name: "PoolBronzePct", Type: "REAL"},
{Name: "PoolSilverPct", Type: "REAL"},
{Name: "PoolGoldPct", Type: "REAL"},
{Name: "Tin", Type: "REAL"},
{Name: "Bronze", Type: "REAL"},
{Name: "Silver", Type: "REAL"},
{Name: "Gold", Type: "REAL"},
}
}
func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error {
if _, err := db.SafeTableName(tableName); err != nil {
return err
}
for _, column := range columns {
if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
return err
}
}
return nil
}
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
errText := strings.ToLower(err.Error())
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
return nil
}
return err
}
return nil
} }
func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error { func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
@@ -600,7 +548,7 @@ func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string)
return err return err
} }
if !hasColumn { if !hasColumn {
if err := addColumnIfMissing(ctx, dbConn, tableName, columnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil { if err := db.AddColumnIfMissing(ctx, dbConn, tableName, db.ColumnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
return err return err
} }
} }
@@ -851,7 +799,11 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
if err := vc.Login(url); err != nil { if err := vc.Login(url); err != nil {
return fmt.Errorf("unable to connect to vcenter: %w", err) return fmt.Errorf("unable to connect to vcenter: %w", err)
} }
defer vc.Logout() defer func() {
if err := vc.Logout(); err != nil {
c.Logger.Warn("vcenter logout failed", "url", url, "error", err)
}
}()
vcVms, err := vc.GetAllVmReferences() vcVms, err := vc.GetAllVmReferences()
if err != nil { if err != nil {

View File

@@ -112,6 +112,7 @@ func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, sna
entries = append(entries, views.SnapshotEntry{ entries = append(entries, views.SnapshotEntry{
Label: label, Label: label,
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName), Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
Count: record.SnapshotCount,
}) })
} }