add record size to hourly snapshot page
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-14 19:43:32 +11:00
parent 9be3a3d807
commit 8df1d145f8
9 changed files with 220 additions and 165 deletions

View File

@@ -7,6 +7,7 @@ import (
type SnapshotEntry struct {
Label string
Link string
Count int64
}
templ SnapshotHourlyList(entries []SnapshotEntry) {
@@ -46,7 +47,10 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {
<ul class="mt-6 space-y-3 web2-list">
for _, entry := range entries {
<li class="flex items-center justify-between gap-4">
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
<div class="flex flex-col">
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
<span class="text-xs text-slate-500">{entry.Count} records</span>
</div>
<a class="web2-link" href={entry.Link}>Download XLSX</a>
</li>
}

View File

@@ -15,6 +15,7 @@ import (
type SnapshotEntry struct {
Label string
Link string
Count int64
}
func SnapshotHourlyList(entries []SnapshotEntry) templ.Component {
@@ -140,7 +141,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 49}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
@@ -153,7 +154,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 36, Col: 55}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
@@ -166,7 +167,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 45, Col: 83}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
@@ -177,38 +178,51 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
return templ_7745c5c3_Err
}
for _, entry := range entries {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><span class=\"text-sm font-semibold text-slate-700\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><div class=\"flex flex-col\"><span class=\"text-sm font-semibold text-slate-700\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 51, Col: 72}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <a class=\"web2-link\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <span class=\"text-xs text-slate-500\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 templ.SafeURL
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Count)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 52, Col: 58}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">Download XLSX</a></li>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, " records</span></div><a class=\"web2-link\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 templ.SafeURL
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 54, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\">Download XLSX</a></li>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</ul></section></main></body>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</ul></section></main></body>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -216,7 +230,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}

View File

@@ -24,6 +24,48 @@ type ColumnDef struct {
Type string
}
// TableRowCount returns COUNT(*) for a table.
func TableRowCount(ctx context.Context, dbConn *sqlx.DB, table string) (int64, error) {
if err := ValidateTableName(table); err != nil {
return 0, err
}
var count int64
query := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, table)
if err := dbConn.GetContext(ctx, &count, query); err != nil {
return 0, err
}
return count, nil
}
// EnsureColumns adds the provided columns to a table if they are missing.
func EnsureColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []ColumnDef) error {
if _, err := SafeTableName(tableName); err != nil {
return err
}
for _, column := range columns {
if err := AddColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
return err
}
}
return nil
}
// AddColumnIfMissing performs a best-effort ALTER TABLE to add a column, ignoring "already exists".
func AddColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column ColumnDef) error {
if _, err := SafeTableName(tableName); err != nil {
return err
}
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
errText := strings.ToLower(err.Error())
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
return nil
}
return err
}
return nil
}
// ValidateTableName ensures table identifiers are safe for interpolation.
func ValidateTableName(name string) error {
if name == "" {

View File

@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE snapshot_registry DROP COLUMN snapshot_count;

View File

@@ -0,0 +1,5 @@
-- +goose Up
ALTER TABLE snapshot_registry ADD COLUMN IF NOT EXISTS snapshot_count BIGINT NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE snapshot_registry DROP COLUMN IF EXISTS snapshot_count;

View File

@@ -70,7 +70,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"snapshot_type" TEXT NOT NULL,
"table_name" TEXT NOT NULL UNIQUE,
"snapshot_time" INTEGER NOT NULL
"snapshot_time" INTEGER NOT NULL,
"snapshot_count" BIGINT NOT NULL DEFAULT 0
);
-- The following tables are declared for sqlc type-checking only.

View File

@@ -16,9 +16,10 @@ import (
)
type SnapshotRecord struct {
TableName string
SnapshotTime time.Time
SnapshotType string
TableName string
SnapshotTime time.Time
SnapshotType string
SnapshotCount int64
}
type SnapshotMigrationStats struct {
@@ -84,20 +85,36 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
)
`)
return err
if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
return err
}
return nil
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS snapshot_registry (
id BIGSERIAL PRIMARY KEY,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
)
`)
return err
if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
return err
}
return nil
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
}
@@ -162,7 +179,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.HourlyRenamed++
}
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime, rowCount); err != nil {
stats.Errors++
continue
}
@@ -180,7 +198,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "daily", table, parsed, rowCount); err != nil {
stats.Errors++
continue
}
@@ -198,7 +217,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed, rowCount); err != nil {
stats.Errors++
continue
}
@@ -211,7 +231,7 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
return stats, nil
}
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time, snapshotCount int64) error {
if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty")
}
@@ -220,16 +240,23 @@ func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType st
switch driver {
case "sqlite":
_, err := dbConn.ExecContext(ctx, `
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES (?, ?, ?)
`, snapshotType, tableName, snapshotTime.Unix())
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES (?, ?, ?, ?)
ON CONFLICT(table_name) DO UPDATE SET
snapshot_time = excluded.snapshot_time,
snapshot_type = excluded.snapshot_type,
snapshot_count = excluded.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES ($1, $2, $3)
ON CONFLICT (table_name) DO NOTHING
`, snapshotType, tableName, snapshotTime.Unix())
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT (table_name) DO UPDATE SET
snapshot_time = EXCLUDED.snapshot_time,
snapshot_type = EXCLUDED.snapshot_type,
snapshot_count = EXCLUDED.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
@@ -264,14 +291,14 @@ func ListSnapshots(ctx context.Context, database db.Database, snapshotType strin
switch driver {
case "sqlite":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = ?
ORDER BY snapshot_time DESC, table_name DESC
`, snapshotType)
case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = $1
ORDER BY snapshot_time DESC, table_name DESC
@@ -291,14 +318,16 @@ ORDER BY snapshot_time DESC, table_name DESC
tableName string
snapshotTime int64
recordType string
snapshotCnt int64
)
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err
}
records = append(records, SnapshotRecord{
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
SnapshotCount: snapshotCnt,
})
}
return records, rows.Err()
@@ -317,7 +346,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
switch driver {
case "sqlite":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = ?
AND snapshot_time >= ?
@@ -326,7 +355,7 @@ ORDER BY snapshot_time ASC, table_name ASC
`, snapshotType, startUnix, endUnix)
case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = $1
AND snapshot_time >= $2
@@ -348,14 +377,16 @@ ORDER BY snapshot_time ASC, table_name ASC
tableName string
snapshotTime int64
recordType string
snapshotCnt int64
)
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err
}
records = append(records, SnapshotRecord{
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
SnapshotCount: snapshotCnt,
})
}
return records, rows.Err()

View File

@@ -86,9 +86,6 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil {
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
}
var wg sync.WaitGroup
var errCount int64
@@ -102,6 +99,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
wg.Add(1)
go func(url string) {
defer wg.Done()
vcStart := time.Now()
if sem != nil {
sem <- struct{}{}
defer func() { <-sem }()
@@ -110,6 +108,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil {
atomic.AddInt64(&errCount, 1)
c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
} else {
c.Logger.Info("Finished hourly snapshot for vcenter", "url", url, "duration", time.Since(vcStart))
}
}(url)
}
@@ -118,7 +118,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount)
}
c.Logger.Debug("Finished hourly vcenter snapshot")
rowCount, err := db.TableRowCount(ctx, dbConn, tableName)
if err != nil {
c.Logger.Warn("unable to count hourly snapshot rows", "error", err, "table", tableName)
}
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime, rowCount); err != nil {
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
}
c.Logger.Debug("Finished hourly vcenter snapshot", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "table", tableName, "row_count", rowCount)
return nil
}
@@ -161,12 +169,6 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
return err
}
}
if rowsExist, err := db.TableHasRows(ctx, dbConn, summaryTable); err != nil {
return err
} else if rowsExist {
c.Logger.Debug("Daily summary already exists, skipping aggregation", "summary_table", summaryTable)
return nil
}
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd)
if err != nil {
@@ -181,12 +183,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
for _, snapshot := range hourlySnapshots {
hourlyTables = append(hourlyTables, snapshot.TableName)
}
unionQuery := buildUnionQuery(hourlyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
unionQuery, err := buildUnionQuery(hourlyTables, summaryUnionColumns, templateExclusionFilter())
if err != nil {
return err
}
currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
if err != nil {
@@ -210,24 +210,23 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
for _, snapshot := range prevSnapshots {
prevTables = append(prevTables, snapshot.TableName)
}
prevUnion := buildUnionQuery(prevTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
if err != nil {
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
prevUnion, err := buildUnionQuery(prevTables, summaryUnionColumns, templateExclusionFilter())
if err == nil {
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
if err != nil {
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
} else {
c.Logger.Info("Daily snapshot comparison",
"current_date", dayStart.Format("2006-01-02"),
"previous_date", prevStart.Format("2006-01-02"),
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
)
}
} else {
c.Logger.Info("Daily snapshot comparison",
"current_date", dayStart.Format("2006-01-02"),
"previous_date", prevStart.Format("2006-01-02"),
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
)
c.Logger.Warn("unable to build previous day union", "error", err)
}
}
@@ -240,7 +239,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart); err != nil {
rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
if err != nil {
c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable)
}
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart, rowCount); err != nil {
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
}
@@ -300,25 +303,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
return err
}
}
if rowsExist, err := db.TableHasRows(ctx, dbConn, monthlyTable); err != nil {
return err
} else if rowsExist {
c.Logger.Debug("Monthly summary already exists, skipping aggregation", "summary_table", monthlyTable)
return nil
}
dailyTables := make([]string, 0, len(dailySnapshots))
for _, snapshot := range dailySnapshots {
dailyTables = append(dailyTables, snapshot.TableName)
}
unionQuery := buildUnionQuery(dailyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
if strings.TrimSpace(unionQuery) == "" {
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
unionQuery, err := buildUnionQuery(dailyTables, summaryUnionColumns, templateExclusionFilter())
if err != nil {
return err
}
monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
@@ -343,7 +335,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil {
rowCount, err := db.TableRowCount(ctx, dbConn, monthlyTable)
if err != nil {
c.Logger.Warn("unable to count monthly summary rows", "error", err, "table", monthlyTable)
}
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth, rowCount); err != nil {
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
}
@@ -443,25 +439,36 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
return db.EnsureColumns(ctx, dbConn, tableName, []db.ColumnDef{
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
})
}
func buildUnionQuery(tables []string, columns []string, whereClause string) string {
func buildUnionQuery(tables []string, columns []string, whereClause string) (string, error) {
if len(tables) == 0 {
return "", fmt.Errorf("no tables provided for union")
}
if len(columns) == 0 {
return "", fmt.Errorf("no columns provided for union")
}
queries := make([]string, 0, len(tables))
columnList := strings.Join(columns, ", ")
for _, table := range tables {
if _, err := db.SafeTableName(table); err != nil {
continue
safeName, err := db.SafeTableName(table)
if err != nil {
return "", err
}
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table)
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
}
queries = append(queries, query)
}
return strings.Join(queries, "\nUNION ALL\n")
if len(queries) == 0 {
return "", fmt.Errorf("no valid tables provided for union")
}
return strings.Join(queries, "\nUNION ALL\n"), nil
}
func templateExclusionFilter() string {
@@ -525,70 +532,11 @@ type columnDef struct {
Type string
}
func summaryMetricColumns() []columnDef {
return []columnDef{
{Name: "InventoryId", Type: "BIGINT"},
{Name: "Name", Type: "TEXT"},
{Name: "Vcenter", Type: "TEXT"},
{Name: "VmId", Type: "TEXT"},
{Name: "EventKey", Type: "TEXT"},
{Name: "CloudId", Type: "TEXT"},
{Name: "CreationTime", Type: "BIGINT"},
{Name: "DeletionTime", Type: "BIGINT"},
{Name: "ResourcePool", Type: "TEXT"},
{Name: "Datacenter", Type: "TEXT"},
{Name: "Cluster", Type: "TEXT"},
{Name: "Folder", Type: "TEXT"},
{Name: "ProvisionedDisk", Type: "REAL"},
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
{Name: "IsTemplate", Type: "TEXT"},
{Name: "PoweredOn", Type: "TEXT"},
{Name: "SrmPlaceholder", Type: "TEXT"},
{Name: "VmUuid", Type: "TEXT"},
{Name: "SamplesPresent", Type: "BIGINT"},
}
}
func summaryAvgColumns() []columnDef {
return []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
{Name: "AvgProvisionedDisk", Type: "REAL"},
{Name: "AvgIsPresent", Type: "REAL"},
{Name: "PoolTinPct", Type: "REAL"},
{Name: "PoolBronzePct", Type: "REAL"},
{Name: "PoolSilverPct", Type: "REAL"},
{Name: "PoolGoldPct", Type: "REAL"},
{Name: "Tin", Type: "REAL"},
{Name: "Bronze", Type: "REAL"},
{Name: "Silver", Type: "REAL"},
{Name: "Gold", Type: "REAL"},
}
}
func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error {
if _, err := db.SafeTableName(tableName); err != nil {
return err
}
for _, column := range columns {
if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
return err
}
}
return nil
}
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
errText := strings.ToLower(err.Error())
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
return nil
}
return err
}
return nil
var summaryUnionColumns = []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}
func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
@@ -600,7 +548,7 @@ func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string)
return err
}
if !hasColumn {
if err := addColumnIfMissing(ctx, dbConn, tableName, columnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
if err := db.AddColumnIfMissing(ctx, dbConn, tableName, db.ColumnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
return err
}
}
@@ -851,7 +799,11 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
if err := vc.Login(url); err != nil {
return fmt.Errorf("unable to connect to vcenter: %w", err)
}
defer vc.Logout()
defer func() {
if err := vc.Logout(); err != nil {
c.Logger.Warn("vcenter logout failed", "url", url, "error", err)
}
}()
vcVms, err := vc.GetAllVmReferences()
if err != nil {

View File

@@ -112,6 +112,7 @@ func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, sna
entries = append(entries, views.SnapshotEntry{
Label: label,
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
Count: record.SnapshotCount,
})
}