add record size to hourly snapshot page
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
|||||||
type SnapshotEntry struct {
|
type SnapshotEntry struct {
|
||||||
Label string
|
Label string
|
||||||
Link string
|
Link string
|
||||||
|
Count int64
|
||||||
}
|
}
|
||||||
|
|
||||||
templ SnapshotHourlyList(entries []SnapshotEntry) {
|
templ SnapshotHourlyList(entries []SnapshotEntry) {
|
||||||
@@ -46,7 +47,10 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {
|
|||||||
<ul class="mt-6 space-y-3 web2-list">
|
<ul class="mt-6 space-y-3 web2-list">
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
<li class="flex items-center justify-between gap-4">
|
<li class="flex items-center justify-between gap-4">
|
||||||
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
|
<div class="flex flex-col">
|
||||||
|
<span class="text-sm font-semibold text-slate-700">{entry.Label}</span>
|
||||||
|
<span class="text-xs text-slate-500">{entry.Count} records</span>
|
||||||
|
</div>
|
||||||
<a class="web2-link" href={entry.Link}>Download XLSX</a>
|
<a class="web2-link" href={entry.Link}>Download XLSX</a>
|
||||||
</li>
|
</li>
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
type SnapshotEntry struct {
|
type SnapshotEntry struct {
|
||||||
Label string
|
Label string
|
||||||
Link string
|
Link string
|
||||||
|
Count int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func SnapshotHourlyList(entries []SnapshotEntry) templ.Component {
|
func SnapshotHourlyList(entries []SnapshotEntry) templ.Component {
|
||||||
@@ -140,7 +141,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
|||||||
var templ_7745c5c3_Var5 string
|
var templ_7745c5c3_Var5 string
|
||||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49}
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 49}
|
||||||
}
|
}
|
||||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
@@ -153,7 +154,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
|||||||
var templ_7745c5c3_Var6 string
|
var templ_7745c5c3_Var6 string
|
||||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
|
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55}
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 36, Col: 55}
|
||||||
}
|
}
|
||||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
@@ -166,7 +167,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
|||||||
var templ_7745c5c3_Var7 string
|
var templ_7745c5c3_Var7 string
|
||||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
|
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83}
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 45, Col: 83}
|
||||||
}
|
}
|
||||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
@@ -177,38 +178,51 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
|||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><span class=\"text-sm font-semibold text-slate-700\">")
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<li class=\"flex items-center justify-between gap-4\"><div class=\"flex flex-col\"><span class=\"text-sm font-semibold text-slate-700\">")
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
var templ_7745c5c3_Var8 string
|
var templ_7745c5c3_Var8 string
|
||||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
|
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71}
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 51, Col: 72}
|
||||||
}
|
}
|
||||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <a class=\"web2-link\" href=\"")
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</span> <span class=\"text-xs text-slate-500\">")
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
var templ_7745c5c3_Var9 templ.SafeURL
|
var templ_7745c5c3_Var9 string
|
||||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
|
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Count)
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45}
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 52, Col: 58}
|
||||||
}
|
}
|
||||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "\">Download XLSX</a></li>")
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, " records</span></div><a class=\"web2-link\" href=\"")
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
var templ_7745c5c3_Var10 templ.SafeURL
|
||||||
|
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 54, Col: 45}
|
||||||
|
}
|
||||||
|
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||||
|
if templ_7745c5c3_Err != nil {
|
||||||
|
return templ_7745c5c3_Err
|
||||||
|
}
|
||||||
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\">Download XLSX</a></li>")
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "</ul></section></main></body>")
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</ul></section></main></body>")
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
@@ -216,7 +230,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
|
|||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</html>")
|
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</html>")
|
||||||
if templ_7745c5c3_Err != nil {
|
if templ_7745c5c3_Err != nil {
|
||||||
return templ_7745c5c3_Err
|
return templ_7745c5c3_Err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,48 @@ type ColumnDef struct {
|
|||||||
Type string
|
Type string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TableRowCount returns COUNT(*) for a table.
|
||||||
|
func TableRowCount(ctx context.Context, dbConn *sqlx.DB, table string) (int64, error) {
|
||||||
|
if err := ValidateTableName(table); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
var count int64
|
||||||
|
query := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, table)
|
||||||
|
if err := dbConn.GetContext(ctx, &count, query); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureColumns adds the provided columns to a table if they are missing.
|
||||||
|
func EnsureColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []ColumnDef) error {
|
||||||
|
if _, err := SafeTableName(tableName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, column := range columns {
|
||||||
|
if err := AddColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddColumnIfMissing performs a best-effort ALTER TABLE to add a column, ignoring "already exists".
|
||||||
|
func AddColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column ColumnDef) error {
|
||||||
|
if _, err := SafeTableName(tableName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
|
||||||
|
if _, err := dbConn.ExecContext(ctx, query); err != nil {
|
||||||
|
errText := strings.ToLower(err.Error())
|
||||||
|
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ValidateTableName ensures table identifiers are safe for interpolation.
|
// ValidateTableName ensures table identifiers are safe for interpolation.
|
||||||
func ValidateTableName(name string) error {
|
func ValidateTableName(name string) error {
|
||||||
if name == "" {
|
if name == "" {
|
||||||
|
|||||||
5
db/migrations/20250116101000_snapshot_count.sql
Normal file
5
db/migrations/20250116101000_snapshot_count.sql
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
-- +goose Up
|
||||||
|
ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE snapshot_registry DROP COLUMN snapshot_count;
|
||||||
5
db/migrations_postgres/20250116101000_snapshot_count.sql
Normal file
5
db/migrations_postgres/20250116101000_snapshot_count.sql
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
-- +goose Up
|
||||||
|
ALTER TABLE snapshot_registry ADD COLUMN IF NOT EXISTS snapshot_count BIGINT NOT NULL DEFAULT 0;
|
||||||
|
|
||||||
|
-- +goose Down
|
||||||
|
ALTER TABLE snapshot_registry DROP COLUMN IF EXISTS snapshot_count;
|
||||||
@@ -70,7 +70,8 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
|
|||||||
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
|
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
"snapshot_type" TEXT NOT NULL,
|
"snapshot_type" TEXT NOT NULL,
|
||||||
"table_name" TEXT NOT NULL UNIQUE,
|
"table_name" TEXT NOT NULL UNIQUE,
|
||||||
"snapshot_time" INTEGER NOT NULL
|
"snapshot_time" INTEGER NOT NULL,
|
||||||
|
"snapshot_count" BIGINT NOT NULL DEFAULT 0
|
||||||
);
|
);
|
||||||
|
|
||||||
-- The following tables are declared for sqlc type-checking only.
|
-- The following tables are declared for sqlc type-checking only.
|
||||||
|
|||||||
@@ -16,9 +16,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type SnapshotRecord struct {
|
type SnapshotRecord struct {
|
||||||
TableName string
|
TableName string
|
||||||
SnapshotTime time.Time
|
SnapshotTime time.Time
|
||||||
SnapshotType string
|
SnapshotType string
|
||||||
|
SnapshotCount int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type SnapshotMigrationStats struct {
|
type SnapshotMigrationStats struct {
|
||||||
@@ -84,20 +85,36 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
|
|||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
snapshot_type TEXT NOT NULL,
|
snapshot_type TEXT NOT NULL,
|
||||||
table_name TEXT NOT NULL UNIQUE,
|
table_name TEXT NOT NULL UNIQUE,
|
||||||
snapshot_time BIGINT NOT NULL
|
snapshot_time BIGINT NOT NULL,
|
||||||
|
snapshot_count BIGINT NOT NULL DEFAULT 0
|
||||||
)
|
)
|
||||||
`)
|
`)
|
||||||
return err
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
|
||||||
|
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
case "pgx", "postgres":
|
case "pgx", "postgres":
|
||||||
_, err := dbConn.ExecContext(ctx, `
|
_, err := dbConn.ExecContext(ctx, `
|
||||||
CREATE TABLE IF NOT EXISTS snapshot_registry (
|
CREATE TABLE IF NOT EXISTS snapshot_registry (
|
||||||
id BIGSERIAL PRIMARY KEY,
|
id BIGSERIAL PRIMARY KEY,
|
||||||
snapshot_type TEXT NOT NULL,
|
snapshot_type TEXT NOT NULL,
|
||||||
table_name TEXT NOT NULL UNIQUE,
|
table_name TEXT NOT NULL UNIQUE,
|
||||||
snapshot_time BIGINT NOT NULL
|
snapshot_time BIGINT NOT NULL,
|
||||||
|
snapshot_count BIGINT NOT NULL DEFAULT 0
|
||||||
)
|
)
|
||||||
`)
|
`)
|
||||||
return err
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
|
||||||
|
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
||||||
}
|
}
|
||||||
@@ -162,7 +179,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
|||||||
stats.HourlyRenamed++
|
stats.HourlyRenamed++
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
|
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||||
|
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime, rowCount); err != nil {
|
||||||
stats.Errors++
|
stats.Errors++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -180,7 +198,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
|||||||
stats.Errors++
|
stats.Errors++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
|
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||||
|
if err := RegisterSnapshot(ctx, database, "daily", table, parsed, rowCount); err != nil {
|
||||||
stats.Errors++
|
stats.Errors++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -198,7 +217,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
|||||||
stats.Errors++
|
stats.Errors++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
|
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
|
||||||
|
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed, rowCount); err != nil {
|
||||||
stats.Errors++
|
stats.Errors++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -211,7 +231,7 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
|
|||||||
return stats, nil
|
return stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
|
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time, snapshotCount int64) error {
|
||||||
if snapshotType == "" || tableName == "" {
|
if snapshotType == "" || tableName == "" {
|
||||||
return fmt.Errorf("snapshot type or table name is empty")
|
return fmt.Errorf("snapshot type or table name is empty")
|
||||||
}
|
}
|
||||||
@@ -220,16 +240,23 @@ func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType st
|
|||||||
switch driver {
|
switch driver {
|
||||||
case "sqlite":
|
case "sqlite":
|
||||||
_, err := dbConn.ExecContext(ctx, `
|
_, err := dbConn.ExecContext(ctx, `
|
||||||
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
|
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
|
||||||
VALUES (?, ?, ?)
|
VALUES (?, ?, ?, ?)
|
||||||
`, snapshotType, tableName, snapshotTime.Unix())
|
ON CONFLICT(table_name) DO UPDATE SET
|
||||||
|
snapshot_time = excluded.snapshot_time,
|
||||||
|
snapshot_type = excluded.snapshot_type,
|
||||||
|
snapshot_count = excluded.snapshot_count
|
||||||
|
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
|
||||||
return err
|
return err
|
||||||
case "pgx", "postgres":
|
case "pgx", "postgres":
|
||||||
_, err := dbConn.ExecContext(ctx, `
|
_, err := dbConn.ExecContext(ctx, `
|
||||||
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
|
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
|
||||||
VALUES ($1, $2, $3)
|
VALUES ($1, $2, $3, $4)
|
||||||
ON CONFLICT (table_name) DO NOTHING
|
ON CONFLICT (table_name) DO UPDATE SET
|
||||||
`, snapshotType, tableName, snapshotTime.Unix())
|
snapshot_time = EXCLUDED.snapshot_time,
|
||||||
|
snapshot_type = EXCLUDED.snapshot_type,
|
||||||
|
snapshot_count = EXCLUDED.snapshot_count
|
||||||
|
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
|
||||||
return err
|
return err
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
|
||||||
@@ -264,14 +291,14 @@ func ListSnapshots(ctx context.Context, database db.Database, snapshotType strin
|
|||||||
switch driver {
|
switch driver {
|
||||||
case "sqlite":
|
case "sqlite":
|
||||||
rows, err = dbConn.QueryxContext(ctx, `
|
rows, err = dbConn.QueryxContext(ctx, `
|
||||||
SELECT table_name, snapshot_time, snapshot_type
|
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||||
FROM snapshot_registry
|
FROM snapshot_registry
|
||||||
WHERE snapshot_type = ?
|
WHERE snapshot_type = ?
|
||||||
ORDER BY snapshot_time DESC, table_name DESC
|
ORDER BY snapshot_time DESC, table_name DESC
|
||||||
`, snapshotType)
|
`, snapshotType)
|
||||||
case "pgx", "postgres":
|
case "pgx", "postgres":
|
||||||
rows, err = dbConn.QueryxContext(ctx, `
|
rows, err = dbConn.QueryxContext(ctx, `
|
||||||
SELECT table_name, snapshot_time, snapshot_type
|
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||||
FROM snapshot_registry
|
FROM snapshot_registry
|
||||||
WHERE snapshot_type = $1
|
WHERE snapshot_type = $1
|
||||||
ORDER BY snapshot_time DESC, table_name DESC
|
ORDER BY snapshot_time DESC, table_name DESC
|
||||||
@@ -291,14 +318,16 @@ ORDER BY snapshot_time DESC, table_name DESC
|
|||||||
tableName string
|
tableName string
|
||||||
snapshotTime int64
|
snapshotTime int64
|
||||||
recordType string
|
recordType string
|
||||||
|
snapshotCnt int64
|
||||||
)
|
)
|
||||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
|
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
records = append(records, SnapshotRecord{
|
records = append(records, SnapshotRecord{
|
||||||
TableName: tableName,
|
TableName: tableName,
|
||||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||||
SnapshotType: recordType,
|
SnapshotType: recordType,
|
||||||
|
SnapshotCount: snapshotCnt,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return records, rows.Err()
|
return records, rows.Err()
|
||||||
@@ -317,7 +346,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
|
|||||||
switch driver {
|
switch driver {
|
||||||
case "sqlite":
|
case "sqlite":
|
||||||
rows, err = dbConn.QueryxContext(ctx, `
|
rows, err = dbConn.QueryxContext(ctx, `
|
||||||
SELECT table_name, snapshot_time, snapshot_type
|
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||||
FROM snapshot_registry
|
FROM snapshot_registry
|
||||||
WHERE snapshot_type = ?
|
WHERE snapshot_type = ?
|
||||||
AND snapshot_time >= ?
|
AND snapshot_time >= ?
|
||||||
@@ -326,7 +355,7 @@ ORDER BY snapshot_time ASC, table_name ASC
|
|||||||
`, snapshotType, startUnix, endUnix)
|
`, snapshotType, startUnix, endUnix)
|
||||||
case "pgx", "postgres":
|
case "pgx", "postgres":
|
||||||
rows, err = dbConn.QueryxContext(ctx, `
|
rows, err = dbConn.QueryxContext(ctx, `
|
||||||
SELECT table_name, snapshot_time, snapshot_type
|
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
|
||||||
FROM snapshot_registry
|
FROM snapshot_registry
|
||||||
WHERE snapshot_type = $1
|
WHERE snapshot_type = $1
|
||||||
AND snapshot_time >= $2
|
AND snapshot_time >= $2
|
||||||
@@ -348,14 +377,16 @@ ORDER BY snapshot_time ASC, table_name ASC
|
|||||||
tableName string
|
tableName string
|
||||||
snapshotTime int64
|
snapshotTime int64
|
||||||
recordType string
|
recordType string
|
||||||
|
snapshotCnt int64
|
||||||
)
|
)
|
||||||
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
|
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
records = append(records, SnapshotRecord{
|
records = append(records, SnapshotRecord{
|
||||||
TableName: tableName,
|
TableName: tableName,
|
||||||
SnapshotTime: time.Unix(snapshotTime, 0),
|
SnapshotTime: time.Unix(snapshotTime, 0),
|
||||||
SnapshotType: recordType,
|
SnapshotType: recordType,
|
||||||
|
SnapshotCount: snapshotCnt,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return records, rows.Err()
|
return records, rows.Err()
|
||||||
|
|||||||
@@ -86,9 +86,6 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
|||||||
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
|
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil {
|
|
||||||
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
var errCount int64
|
var errCount int64
|
||||||
@@ -102,6 +99,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func(url string) {
|
go func(url string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
vcStart := time.Now()
|
||||||
if sem != nil {
|
if sem != nil {
|
||||||
sem <- struct{}{}
|
sem <- struct{}{}
|
||||||
defer func() { <-sem }()
|
defer func() { <-sem }()
|
||||||
@@ -110,6 +108,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
|||||||
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil {
|
if err := c.captureHourlySnapshotForVcenter(ctx, startTime, tableName, url); err != nil {
|
||||||
atomic.AddInt64(&errCount, 1)
|
atomic.AddInt64(&errCount, 1)
|
||||||
c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
|
c.Logger.Error("hourly snapshot failed", "error", err, "url", url)
|
||||||
|
} else {
|
||||||
|
c.Logger.Info("Finished hourly snapshot for vcenter", "url", url, "duration", time.Since(vcStart))
|
||||||
}
|
}
|
||||||
}(url)
|
}(url)
|
||||||
}
|
}
|
||||||
@@ -118,7 +118,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
|
|||||||
return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount)
|
return fmt.Errorf("hourly snapshot failed for %d vcenter(s)", errCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Logger.Debug("Finished hourly vcenter snapshot")
|
rowCount, err := db.TableRowCount(ctx, dbConn, tableName)
|
||||||
|
if err != nil {
|
||||||
|
c.Logger.Warn("unable to count hourly snapshot rows", "error", err, "table", tableName)
|
||||||
|
}
|
||||||
|
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime, rowCount); err != nil {
|
||||||
|
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Logger.Debug("Finished hourly vcenter snapshot", "vcenter_count", len(c.Settings.Values.Settings.VcenterAddresses), "table", tableName, "row_count", rowCount)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,12 +169,6 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if rowsExist, err := db.TableHasRows(ctx, dbConn, summaryTable); err != nil {
|
|
||||||
return err
|
|
||||||
} else if rowsExist {
|
|
||||||
c.Logger.Debug("Daily summary already exists, skipping aggregation", "summary_table", summaryTable)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd)
|
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -181,12 +183,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
|||||||
for _, snapshot := range hourlySnapshots {
|
for _, snapshot := range hourlySnapshots {
|
||||||
hourlyTables = append(hourlyTables, snapshot.TableName)
|
hourlyTables = append(hourlyTables, snapshot.TableName)
|
||||||
}
|
}
|
||||||
unionQuery := buildUnionQuery(hourlyTables, []string{
|
unionQuery, err := buildUnionQuery(hourlyTables, summaryUnionColumns, templateExclusionFilter())
|
||||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
if err != nil {
|
||||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
return err
|
||||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
}
|
||||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
|
||||||
}, templateExclusionFilter())
|
|
||||||
|
|
||||||
currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
currentTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -210,24 +210,23 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
|||||||
for _, snapshot := range prevSnapshots {
|
for _, snapshot := range prevSnapshots {
|
||||||
prevTables = append(prevTables, snapshot.TableName)
|
prevTables = append(prevTables, snapshot.TableName)
|
||||||
}
|
}
|
||||||
prevUnion := buildUnionQuery(prevTables, []string{
|
prevUnion, err := buildUnionQuery(prevTables, summaryUnionColumns, templateExclusionFilter())
|
||||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
if err == nil {
|
||||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
|
||||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
if err != nil {
|
||||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
|
||||||
}, templateExclusionFilter())
|
} else {
|
||||||
prevTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, prevUnion)
|
c.Logger.Info("Daily snapshot comparison",
|
||||||
if err != nil {
|
"current_date", dayStart.Format("2006-01-02"),
|
||||||
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
|
"previous_date", prevStart.Format("2006-01-02"),
|
||||||
|
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
|
||||||
|
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
|
||||||
|
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
|
||||||
|
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
|
||||||
|
)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
c.Logger.Info("Daily snapshot comparison",
|
c.Logger.Warn("unable to build previous day union", "error", err)
|
||||||
"current_date", dayStart.Format("2006-01-02"),
|
|
||||||
"previous_date", prevStart.Format("2006-01-02"),
|
|
||||||
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
|
|
||||||
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
|
|
||||||
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
|
|
||||||
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -240,7 +239,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
|
|||||||
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
|
c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02"))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart); err != nil {
|
rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
|
||||||
|
if err != nil {
|
||||||
|
c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable)
|
||||||
|
}
|
||||||
|
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, dayStart, rowCount); err != nil {
|
||||||
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
|
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -300,25 +303,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if rowsExist, err := db.TableHasRows(ctx, dbConn, monthlyTable); err != nil {
|
|
||||||
return err
|
|
||||||
} else if rowsExist {
|
|
||||||
c.Logger.Debug("Monthly summary already exists, skipping aggregation", "summary_table", monthlyTable)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
dailyTables := make([]string, 0, len(dailySnapshots))
|
dailyTables := make([]string, 0, len(dailySnapshots))
|
||||||
for _, snapshot := range dailySnapshots {
|
for _, snapshot := range dailySnapshots {
|
||||||
dailyTables = append(dailyTables, snapshot.TableName)
|
dailyTables = append(dailyTables, snapshot.TableName)
|
||||||
}
|
}
|
||||||
unionQuery := buildUnionQuery(dailyTables, []string{
|
unionQuery, err := buildUnionQuery(dailyTables, summaryUnionColumns, templateExclusionFilter())
|
||||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
if err != nil {
|
||||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
return err
|
||||||
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
|
||||||
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
|
||||||
}, templateExclusionFilter())
|
|
||||||
if strings.TrimSpace(unionQuery) == "" {
|
|
||||||
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
monthlyTotals, err := db.SnapshotTotalsForUnion(ctx, dbConn, unionQuery)
|
||||||
@@ -343,7 +335,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
|
|||||||
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
|
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil {
|
rowCount, err := db.TableRowCount(ctx, dbConn, monthlyTable)
|
||||||
|
if err != nil {
|
||||||
|
c.Logger.Warn("unable to count monthly summary rows", "error", err, "table", monthlyTable)
|
||||||
|
}
|
||||||
|
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth, rowCount); err != nil {
|
||||||
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
|
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -443,25 +439,36 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
|
return db.EnsureColumns(ctx, dbConn, tableName, []db.ColumnDef{
|
||||||
{Name: "VcpuCount", Type: "BIGINT"},
|
{Name: "VcpuCount", Type: "BIGINT"},
|
||||||
{Name: "RamGB", Type: "BIGINT"},
|
{Name: "RamGB", Type: "BIGINT"},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
func buildUnionQuery(tables []string, columns []string, whereClause string) string {
|
func buildUnionQuery(tables []string, columns []string, whereClause string) (string, error) {
|
||||||
|
if len(tables) == 0 {
|
||||||
|
return "", fmt.Errorf("no tables provided for union")
|
||||||
|
}
|
||||||
|
if len(columns) == 0 {
|
||||||
|
return "", fmt.Errorf("no columns provided for union")
|
||||||
|
}
|
||||||
|
|
||||||
queries := make([]string, 0, len(tables))
|
queries := make([]string, 0, len(tables))
|
||||||
columnList := strings.Join(columns, ", ")
|
columnList := strings.Join(columns, ", ")
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
if _, err := db.SafeTableName(table); err != nil {
|
safeName, err := db.SafeTableName(table)
|
||||||
continue
|
if err != nil {
|
||||||
|
return "", err
|
||||||
}
|
}
|
||||||
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table)
|
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
|
||||||
if whereClause != "" {
|
if whereClause != "" {
|
||||||
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
|
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
|
||||||
}
|
}
|
||||||
queries = append(queries, query)
|
queries = append(queries, query)
|
||||||
}
|
}
|
||||||
return strings.Join(queries, "\nUNION ALL\n")
|
if len(queries) == 0 {
|
||||||
|
return "", fmt.Errorf("no valid tables provided for union")
|
||||||
|
}
|
||||||
|
return strings.Join(queries, "\nUNION ALL\n"), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func templateExclusionFilter() string {
|
func templateExclusionFilter() string {
|
||||||
@@ -525,70 +532,11 @@ type columnDef struct {
|
|||||||
Type string
|
Type string
|
||||||
}
|
}
|
||||||
|
|
||||||
func summaryMetricColumns() []columnDef {
|
var summaryUnionColumns = []string{
|
||||||
return []columnDef{
|
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||||
{Name: "InventoryId", Type: "BIGINT"},
|
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||||
{Name: "Name", Type: "TEXT"},
|
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
|
||||||
{Name: "Vcenter", Type: "TEXT"},
|
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
|
||||||
{Name: "VmId", Type: "TEXT"},
|
|
||||||
{Name: "EventKey", Type: "TEXT"},
|
|
||||||
{Name: "CloudId", Type: "TEXT"},
|
|
||||||
{Name: "CreationTime", Type: "BIGINT"},
|
|
||||||
{Name: "DeletionTime", Type: "BIGINT"},
|
|
||||||
{Name: "ResourcePool", Type: "TEXT"},
|
|
||||||
{Name: "Datacenter", Type: "TEXT"},
|
|
||||||
{Name: "Cluster", Type: "TEXT"},
|
|
||||||
{Name: "Folder", Type: "TEXT"},
|
|
||||||
{Name: "ProvisionedDisk", Type: "REAL"},
|
|
||||||
{Name: "VcpuCount", Type: "BIGINT"},
|
|
||||||
{Name: "RamGB", Type: "BIGINT"},
|
|
||||||
{Name: "IsTemplate", Type: "TEXT"},
|
|
||||||
{Name: "PoweredOn", Type: "TEXT"},
|
|
||||||
{Name: "SrmPlaceholder", Type: "TEXT"},
|
|
||||||
{Name: "VmUuid", Type: "TEXT"},
|
|
||||||
{Name: "SamplesPresent", Type: "BIGINT"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func summaryAvgColumns() []columnDef {
|
|
||||||
return []columnDef{
|
|
||||||
{Name: "AvgVcpuCount", Type: "REAL"},
|
|
||||||
{Name: "AvgRamGB", Type: "REAL"},
|
|
||||||
{Name: "AvgProvisionedDisk", Type: "REAL"},
|
|
||||||
{Name: "AvgIsPresent", Type: "REAL"},
|
|
||||||
{Name: "PoolTinPct", Type: "REAL"},
|
|
||||||
{Name: "PoolBronzePct", Type: "REAL"},
|
|
||||||
{Name: "PoolSilverPct", Type: "REAL"},
|
|
||||||
{Name: "PoolGoldPct", Type: "REAL"},
|
|
||||||
{Name: "Tin", Type: "REAL"},
|
|
||||||
{Name: "Bronze", Type: "REAL"},
|
|
||||||
{Name: "Silver", Type: "REAL"},
|
|
||||||
{Name: "Gold", Type: "REAL"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error {
|
|
||||||
if _, err := db.SafeTableName(tableName); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, column := range columns {
|
|
||||||
if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
|
|
||||||
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
|
|
||||||
if _, err := dbConn.ExecContext(ctx, query); err != nil {
|
|
||||||
errText := strings.ToLower(err.Error())
|
|
||||||
if strings.Contains(errText, "duplicate column") || strings.Contains(errText, "already exists") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
|
func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
|
||||||
@@ -600,7 +548,7 @@ func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string)
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !hasColumn {
|
if !hasColumn {
|
||||||
if err := addColumnIfMissing(ctx, dbConn, tableName, columnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
|
if err := db.AddColumnIfMissing(ctx, dbConn, tableName, db.ColumnDef{Name: "RowId", Type: "BIGSERIAL"}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -851,7 +799,11 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
|
|||||||
if err := vc.Login(url); err != nil {
|
if err := vc.Login(url); err != nil {
|
||||||
return fmt.Errorf("unable to connect to vcenter: %w", err)
|
return fmt.Errorf("unable to connect to vcenter: %w", err)
|
||||||
}
|
}
|
||||||
defer vc.Logout()
|
defer func() {
|
||||||
|
if err := vc.Logout(); err != nil {
|
||||||
|
c.Logger.Warn("vcenter logout failed", "url", url, "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
vcVms, err := vc.GetAllVmReferences()
|
vcVms, err := vc.GetAllVmReferences()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -112,6 +112,7 @@ func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, sna
|
|||||||
entries = append(entries, views.SnapshotEntry{
|
entries = append(entries, views.SnapshotEntry{
|
||||||
Label: label,
|
Label: label,
|
||||||
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
|
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
|
||||||
|
Count: record.SnapshotCount,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user