add record size to hourly snapshot page
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
type SnapshotEntry struct {
|
||||
Label string
|
||||
Link string
|
||||
Count int64
|
||||
}
|
||||
|
||||
templ SnapshotHourlyList(entries []SnapshotEntry) {
|
||||
@@ -46,7 +47,10 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {
|
||||
<ul class="mt-6 space-y-3 web2-list">
|
||||
for _, entry := range entries {
|
||||
<li class="flex items-center justify-between gap-4">
|
||||
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
|
||||
<div class="flex flex-col">
|
||||
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
|
||||
<span class="text-xs text-slate-500">{entry.Count} records</span>
|
||||
</div>
|
||||
<a class="web2-link" href={entry.Link}>Download XLSX</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
type SnapshotEntry struct {
|
||||
Label string
|
||||
Link string
|
||||
Count int64
|
||||
}
|
||||
|
||||
func SnapshotHourlyList(entries []SnapshotEntry) templ.Component {
|
||||
@@ -140,7 +141,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 49}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -153,7 +154,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 36, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -166,7 +167,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 45, Col: 83}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -177,38 +178,51 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><span class=\"text-sm font-semibold text-slate-700\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><div class=\"flex flex-col\"><span class=\"text-sm font-semibold text-slate-700\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 51, Col: 72}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <a class=\"web2-link\" href=\"")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <span class=\"text-xs text-slate-500\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var9 templ.SafeURL
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Count)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 52, Col: 58}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">Download XLSX</a></li>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, " records</span></div><a class=\"web2-link\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 templ.SafeURL
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 54, Col: 45}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\">Download XLSX</a></li>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</ul></section></main></body>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</ul></section></main></body>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -216,7 +230,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</html>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</html>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -24,6 +24,48 @@ type ColumnDef struct {
|
||||
Type string
|
||||
}
|
||||
|
||||
// TableRowCount returns COUNT(*) for a table.
|
||||
func TableRowCount(ctx context.Context, dbConn *sqlx.DB, table string) (int64, error) {
|
||||
if err := ValidateTableName(table); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var count int64
|
||||
query := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, table)
|
||||
if err := dbConn.GetContext(ctx, &count, query); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// EnsureColumns adds the provided columns to a table if they are missing.
|
||||
func EnsureColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []ColumnDef) error {
|
||||
if _, err := SafeTableName(tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, column := range columns {
|
||||
if err := AddColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddColumnIfMissing performs a best-effort ALTER TABLE to add a column, ignoring "already exists".
|
||||
func AddColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column ColumnDef) error {
|
||||
if _, err := SafeTableName(tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
|
||||
if _, err := dbConn.ExecContext(ctx, query); err != nil {
|
||||
errText := strings.ToLower(err.Error())
|
||||
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateTableName ensures table identifiers are safe for interpolation.
|
||||
func ValidateTableName(name string) error {
|
||||
if name == "" {
|
||||
|
||||
5
db/migrations/20250116101000_snapshot_count.sql
Normal file
5
db/migrations/20250116101000_snapshot_count.sql
Normal file
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE snapshot_registry DROP COLUMN snapshot_count;
|
||||
5
db/migrations_postgres/20250116101000_snapshot_count.sql
Normal file
5
db/migrations_postgres/20250116101000_snapshot_count.sql
Normal file
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE snapshot_registry ADD COLUMN IF NOT EXISTS snapshot_count BIGINT NOT NULL DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE snapshot_registry DROP COLUMN IF EXISTS snapshot_count;
|
||||
@@ -70,7 +70,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
|
||||
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
"snapshot_type" TEXT NOT NULL,
|
||||
"table_name" TEXT NOT NULL UNIQUE,
|
||||
"snapshot_time" INTEGER NOT NULL
|
||||
"snapshot_time" INTEGER NOT NULL,
|
||||
"snapshot_count" BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- The following tables are declared for sqlc type-checking only.
|
||||
|
||||
@@ -16,9 +16,10 @@ import (
|
||||
)
|
||||
|
||||
type SnapshotRecord struct {
|
||||
TableName string
|
||||
SnapshotTime time.Time
|
||||
SnapshotType string
|
||||
TableName string
|
||||
SnapshotTime time.Time
|
||||
SnapshotType string
|
||||
SnapshotCount int64
|
||||
}
|
||||
|
||||
type SnapshotMigrationStats struct {
|
||||
@@ -84,20 +85,36 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
snapshot_type TEXT NOT NULL,
|
||||
table_name TEXT NOT NULL UNIQUE,
|
||||
snapshot_time BIGINT NOT NULL
|
||||
snapshot_time BIGINT NOT NULL,
|
||||
snapshot_count BIGINT NOT NULL DEFAULT 0
|
||||
)
|
||||
`)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
|
||||
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
case "pgx", "postgres":
|
||||
_, err := dbConn.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS snapshot_registry (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
snapshot_type TEXT NOT NULL,
|
||||
table_name TEXT NOT NULL UNIQUE,
|
||||
snapshot_time BIGINT NOT NULL
|
||||
snapshot_time BIGINT NOT NULL,
|
||||
snapshot_count BIGINT NOT NULL DEFAULT 0
|
||||
)
|
||||
`)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
|
||||
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
||||
}
|
||||
@@ -162,7 +179,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
||||
stats.HourlyRenamed++
|
||||
}
|
||||
|
||||
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
|
||||
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime, rowCount); err != nil {
|
||||
stats.Errors++
|
||||
continue
|
||||
}
|
||||
@@ -180,7 +198,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
||||
stats.Errors++
|
||||
continue
|
||||
}
|
||||
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
|
||||
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||
if err := RegisterSnapshot(ctx, database, "daily", table, parsed, rowCount); err != nil {
|
||||
stats.Errors++
|
||||
continue
|
||||
}
|
||||
@@ -198,7 +217,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
||||
stats.Errors++
|
||||
continue
|
||||
}
|
||||
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
|
||||
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed, rowCount); err != nil {
|
||||
stats.Errors++
|
||||
continue
|
||||
}
|
||||
@@ -211,7 +231,7 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
|
||||
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time, snapshotCount int64) error {
|
||||
if snapshotType == "" || tableName == "" {
|
||||
return fmt.Errorf("snapshot type or table name is empty")
|
||||
}
|
||||
@@ -220,16 +240,23 @@ func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType st
|
||||
switch driver {
|
||||
case "sqlite":
|
||||
_, err := dbConn.ExecContext(ctx, `
|
||||
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
|
||||
VALUES (?, ?, ?)
|
||||
`, snapshotType, tableName, snapshotTime.Unix())
|
||||
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(table_name) DO UPDATE SET
|
||||
snapshot_time = excluded.snapshot_time,
|
||||
snapshot_type = excluded.snapshot_type,
|
||||
snapshot_count = excluded.snapshot_count
|
||||
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
|
||||
return err
|
||||
case "pgx", "postgres":
|
||||
_, err := dbConn.ExecContext(ctx, `
|
||||
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (table_name) DO NOTHING
|
||||
`, snapshotType, tableName, snapshotTime.Unix())
|
||||
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
snapshot_time = EXCLUDED.snapshot_time,
|
||||
snapshot_type = EXCLUDED.snapshot_type,
|
||||
snapshot_count = EXCLUDED.snapshot_count
|
||||
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
|
||||
return err
|
||||
default:
|
||||
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
||||
@@ -264,14 +291,14 @@ func ListSnapshots(ctx context.Context, database db.Database, snapshotType strin
|
||||
switch driver {
|
||||
case "sqlite":
|
||||
rows, err = dbConn.QueryxContext(ctx, `
|
||||
SELECT table_name, snapshot_time, snapshot_type
|
||||
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||
FROM snapshot_registry
|
||||
WHERE snapshot_type = ?
|
||||
ORDER BY snapshot_time DESC, table_name DESC
|
||||
`, snapshotType)
|
||||
case "pgx", "postgres":
|
||||
rows, err = dbConn.QueryxContext(ctx, `
|
||||
SELECT table_name, snapshot_time, snapshot_type
|
||||
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||
FROM snapshot_registry
|
||||
WHERE snapshot_type = $1
|
||||
ORDER BY snapshot_time DESC, table_name DESC
|
||||
@@ -291,14 +318,16 @@ ORDER BY snapshot_time DESC, table_name DESC
|
||||
tableName string
|
||||
snapshotTime int64
|
||||
recordType string
|
||||
snapshotCnt int64
|
||||
)
|
||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
|
||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, SnapshotRecord{
|
||||
TableName: tableName,
|
||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||
SnapshotType: recordType,
|
||||
TableName: tableName,
|
||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||
SnapshotType: recordType,
|
||||
SnapshotCount: snapshotCnt,
|
||||
})
|
||||
}
|
||||
return records, rows.Err()
|
||||
@@ -317,7 +346,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
|
||||
switch driver {
|
||||
case "sqlite":
|
||||
rows, err = dbConn.QueryxContext(ctx, `
|
||||
SELECT table_name, snapshot_time, snapshot_type
|
||||
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||
FROM snapshot_registry
|
||||
WHERE snapshot_type = ?
|
||||
AND snapshot_time >= ?
|
||||
@@ -326,7 +355,7 @@ ORDER BY snapshot_time ASC, table_name ASC
|
||||
`, snapshotType, startUnix, endUnix)
|
||||
case "pgx", "postgres":
|
||||
rows, err = dbConn.QueryxContext(ctx, `
|
||||
SELECT table_name, snapshot_time, snapshot_type
|
||||
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||
FROM snapshot_registry
|
||||
WHERE snapshot_type = $1
|
||||
AND snapshot_time >= $2
|
||||
@@ -348,14 +377,16 @@ ORDER BY snapshot_time ASC, table_name ASC
|
||||
tableName string
|
||||
snapshotTime int64
|
||||
recordType string
|
||||
snapshotCnt int64
|
||||
)
|
||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
|
||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, SnapshotRecord{
|
||||
TableName: tableName,
|
||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||
SnapshotType: recordType,
|
||||
TableName: tableName,
|
||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||
SnapshotType: recordType,
|
||||
SnapshotCount: snapshotCnt,
|
||||
})
|
||||
}
|
||||
return records, rows.Err()
|
||||
|
||||
@@ -86,9 +86,6 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
||||
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil {
|
||||
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var errCount int64
|
||||
@@ -102,6 +99,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
||||
wg.Add(1)
|
||||
go func(url string) {
|
||||
defer wg.Done()
|
||||
vcStart := time.Now()
|
||||
if sem != nil {
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
@@ -110,6 +108,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
||||
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil {
|
||||
atomic.AddInt64(&errCount, 1)
|
||||
c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
|
||||
} else {
|
||||
c.Logger.Info("Finished hourly snapshot for vcenter", "url", url, "duration", time.Since(vcStart))
|
||||
}
|
||||
}(url)
|
||||
}
|
||||
@@ -118,7 +118,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
||||
return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount)
|
||||
}
|
||||
|
||||
c.Logger.Debug("Finished hourly vcenter snapshot")
|
||||
rowCount, err := db.TableRowCount(ctx, dbConn, tableName)
|
||||
if err != nil {
|
||||
c.Logger.Warn("unable to count hourly snapshot rows", "error", err, "table", tableName)
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime, rowCount); err != nil {
|
||||
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
||||
}
|
||||
|
||||
c.Logger.Debug("Finished hourly vcenter snapshot", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "table", tableName, "row_count", rowCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -161,12 +169,6 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
||||
return err
|
||||
}
|
||||
}
|
||||
if rowsExist, err := db.TableHasRows(ctx, dbConn, summaryTable); err != nil {
|
||||
return err
|
||||
} else if rowsExist {
|
||||
c.Logger.Debug("Daily summary already exists, skipping aggregation", "summary_table", summaryTable)
|
||||
return nil
|
||||
}
|
||||
|
||||
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd)
|
||||
if err != nil {
|
||||
@@ -181,12 +183,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
||||
for _, snapshot := range hourlySnapshots {
|
||||
hourlyTables = append(hourlyTables, snapshot.TableName)
|
||||
}
|
||||
unionQuery := buildUnionQuery(hourlyTables, []string{
|
||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
||||
}, templateExclusionFilter())
|
||||
unionQuery, err := buildUnionQuery(hourlyTables, summaryUnionColumns, templateExclusionFilter())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
||||
if err != nil {
|
||||
@@ -210,24 +210,23 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
||||
for _, snapshot := range prevSnapshots {
|
||||
prevTables = append(prevTables, snapshot.TableName)
|
||||
}
|
||||
prevUnion := buildUnionQuery(prevTables, []string{
|
||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
||||
}, templateExclusionFilter())
|
||||
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
|
||||
if err != nil {
|
||||
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
|
||||
prevUnion, err := buildUnionQuery(prevTables, summaryUnionColumns, templateExclusionFilter())
|
||||
if err == nil {
|
||||
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
|
||||
if err != nil {
|
||||
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
|
||||
} else {
|
||||
c.Logger.Info("Daily snapshot comparison",
|
||||
"current_date", dayStart.Format("2006-01-02"),
|
||||
"previous_date", prevStart.Format("2006-01-02"),
|
||||
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
|
||||
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
|
||||
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
|
||||
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
c.Logger.Info("Daily snapshot comparison",
|
||||
"current_date", dayStart.Format("2006-01-02"),
|
||||
"previous_date", prevStart.Format("2006-01-02"),
|
||||
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
|
||||
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
|
||||
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
|
||||
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
|
||||
)
|
||||
c.Logger.Warn("unable to build previous day union", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,7 +239,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
||||
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
|
||||
return err
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart); err != nil {
|
||||
rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
|
||||
if err != nil {
|
||||
c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable)
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart, rowCount); err != nil {
|
||||
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
|
||||
}
|
||||
|
||||
@@ -300,25 +303,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
|
||||
return err
|
||||
}
|
||||
}
|
||||
if rowsExist, err := db.TableHasRows(ctx, dbConn, monthlyTable); err != nil {
|
||||
return err
|
||||
} else if rowsExist {
|
||||
c.Logger.Debug("Monthly summary already exists, skipping aggregation", "summary_table", monthlyTable)
|
||||
return nil
|
||||
}
|
||||
|
||||
dailyTables := make([]string, 0, len(dailySnapshots))
|
||||
for _, snapshot := range dailySnapshots {
|
||||
dailyTables = append(dailyTables, snapshot.TableName)
|
||||
}
|
||||
unionQuery := buildUnionQuery(dailyTables, []string{
|
||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
||||
}, templateExclusionFilter())
|
||||
if strings.TrimSpace(unionQuery) == "" {
|
||||
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
|
||||
unionQuery, err := buildUnionQuery(dailyTables, summaryUnionColumns, templateExclusionFilter())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
||||
@@ -343,7 +335,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
|
||||
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
|
||||
return err
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil {
|
||||
rowCount, err := db.TableRowCount(ctx, dbConn, monthlyTable)
|
||||
if err != nil {
|
||||
c.Logger.Warn("unable to count monthly summary rows", "error", err, "table", monthlyTable)
|
||||
}
|
||||
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth, rowCount); err != nil {
|
||||
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
|
||||
}
|
||||
|
||||
@@ -443,25 +439,36 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
|
||||
return err
|
||||
}
|
||||
|
||||
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
|
||||
return db.EnsureColumns(ctx, dbConn, tableName, []db.ColumnDef{
|
||||
{Name: "VcpuCount", Type: "BIGINT"},
|
||||
{Name: "RamGB", Type: "BIGINT"},
|
||||
})
|
||||
}
|
||||
func buildUnionQuery(tables []string, columns []string, whereClause string) string {
|
||||
func buildUnionQuery(tables []string, columns []string, whereClause string) (string, error) {
|
||||
if len(tables) == 0 {
|
||||
return "", fmt.Errorf("no tables provided for union")
|
||||
}
|
||||
if len(columns) == 0 {
|
||||
return "", fmt.Errorf("no columns provided for union")
|
||||
}
|
||||
|
||||
queries := make([]string, 0, len(tables))
|
||||
columnList := strings.Join(columns, ", ")
|
||||
for _, table := range tables {
|
||||
if _, err := db.SafeTableName(table); err != nil {
|
||||
continue
|
||||
safeName, err := db.SafeTableName(table)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table)
|
||||
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
|
||||
if whereClause != "" {
|
||||
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
|
||||
}
|
||||
queries = append(queries, query)
|
||||
}
|
||||
return strings.Join(queries, "\nUNION ALL\n")
|
||||
if len(queries) == 0 {
|
||||
return "", fmt.Errorf("no valid tables provided for union")
|
||||
}
|
||||
return strings.Join(queries, "\nUNION ALL\n"), nil
|
||||
}
|
||||
|
||||
func templateExclusionFilter() string {
|
||||
@@ -525,70 +532,11 @@ type columnDef struct {
|
||||
Type string
|
||||
}
|
||||
|
||||
func summaryMetricColumns() []columnDef {
|
||||
return []columnDef{
|
||||
{Name: "InventoryId", Type: "BIGINT"},
|
||||
{Name: "Name", Type: "TEXT"},
|
||||
{Name: "Vcenter", Type: "TEXT"},
|
||||
{Name: "VmId", Type: "TEXT"},
|
||||
{Name: "EventKey", Type: "TEXT"},
|
||||
{Name: "CloudId", Type: "TEXT"},
|
||||
{Name: "CreationTime", Type: "BIGINT"},
|
||||
{Name: "DeletionTime", Type: "BIGINT"},
|
||||
{Name: "ResourcePool", Type: "TEXT"},
|
||||
{Name: "Datacenter", Type: "TEXT"},
|
||||
{Name: "Cluster", Type: "TEXT"},
|
||||
{Name: "Folder", Type: "TEXT"},
|
||||
{Name: "ProvisionedDisk", Type: "REAL"},
|
||||
{Name: "VcpuCount", Type: "BIGINT"},
|
||||
{Name: "RamGB", Type: "BIGINT"},
|
||||
{Name: "IsTemplate", Type: "TEXT"},
|
||||
{Name: "PoweredOn", Type: "TEXT"},
|
||||
{Name: "SrmPlaceholder", Type: "TEXT"},
|
||||
{Name: "VmUuid", Type: "TEXT"},
|
||||
{Name: "SamplesPresent", Type: "BIGINT"},
|
||||
}
|
||||
}
|
||||
|
||||
func summaryAvgColumns() []columnDef {
|
||||
return []columnDef{
|
||||
{Name: "AvgVcpuCount", Type: "REAL"},
|
||||
{Name: "AvgRamGB", Type: "REAL"},
|
||||
{Name: "AvgProvisionedDisk", Type: "REAL"},
|
||||
{Name: "AvgIsPresent", Type: "REAL"},
|
||||
{Name: "PoolTinPct", Type: "REAL"},
|
||||
{Name: "PoolBronzePct", Type: "REAL"},
|
||||
{Name: "PoolSilverPct", Type: "REAL"},
|
||||
{Name: "PoolGoldPct", Type: "REAL"},
|
||||
{Name: "Tin", Type: "REAL"},
|
||||
{Name: "Bronze", Type: "REAL"},
|
||||
{Name: "Silver", Type: "REAL"},
|
||||
{Name: "Gold", Type: "REAL"},
|
||||
}
|
||||
}
|
||||
|
||||
func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error {
|
||||
if _, err := db.SafeTableName(tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, column := range columns {
|
||||
if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
|
||||
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
|
||||
if _, err := dbConn.ExecContext(ctx, query); err != nil {
|
||||
errText := strings.ToLower(err.Error())
|
||||
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
var summaryUnionColumns = []string{
|
||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
||||
}
|
||||
|
||||
func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
|
||||
@@ -600,7 +548,7 @@ func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string)
|
||||
return err
|
||||
}
|
||||
if !hasColumn {
|
||||
if err := addColumnIfMissing(ctx, dbConn, tableName, columnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
|
||||
if err := db.AddColumnIfMissing(ctx, dbConn, tableName, db.ColumnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -851,7 +799,11 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
||||
if err := vc.Login(url); err != nil {
|
||||
return fmt.Errorf("unable to connect to vcenter: %w", err)
|
||||
}
|
||||
defer vc.Logout()
|
||||
defer func() {
|
||||
if err := vc.Logout(); err != nil {
|
||||
c.Logger.Warn("vcenter logout failed", "url", url, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
vcVms, err := vc.GetAllVmReferences()
|
||||
if err != nil {
|
||||
|
||||
@@ -112,6 +112,7 @@ func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, sna
|
||||
entries = append(entries, views.SnapshotEntry{
|
||||
Label: label,
|
||||
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
|
||||
Count: record.SnapshotCount,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user