nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

avoid vcenter totals pages scanning whole database
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-02-09 13:44:43 +11:00
parent c66679a71f
commit 5736dc6929
11 changed files with 991 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
db/helpers_cache_and_index_test.go
View File

@@ -162,3 +162,316 @@ func TestParseHourlySnapshotUnix(t *testing.T) {
}
}
}
func TestVcenterLatestTotalsAndListFallback(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
t.Fatalf("failed to ensure vcenter_latest_totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", time.Unix(200, 0), 10, 20, 30); err != nil {
t.Fatalf("failed to insert totals for vc-a: %v", err)
}
// Older snapshot should not replace latest totals.
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", time.Unix(100, 0), 1, 2, 3); err != nil {
t.Fatalf("failed to insert older totals for vc-a: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-b", time.Unix(300, 0), 11, 21, 31); err != nil {
t.Fatalf("failed to insert totals for vc-b: %v", err)
}
vcenters, err := ListVcenters(ctx, dbConn)
if err != nil {
t.Fatalf("ListVcenters failed: %v", err)
}
if len(vcenters) != 2 || vcenters[0] != "vc-a" || vcenters[1] != "vc-b" {
t.Fatalf("unexpected vcenter list: %#v", vcenters)
}
var latest struct {
SnapshotTime int64 `db:"SnapshotTime"`
VmCount int64 `db:"VmCount"`
VcpuTotal int64 `db:"VcpuTotal"`
RamTotalGB int64 `db:"RamTotalGB"`
}
if err := dbConn.GetContext(ctx, &latest, `
SELECT "SnapshotTime","VmCount","VcpuTotal","RamTotalGB"
FROM vcenter_latest_totals
WHERE "Vcenter" = ?
`, "vc-a"); err != nil {
t.Fatalf("failed to query latest totals for vc-a: %v", err)
}
if latest.SnapshotTime != 200 || latest.VmCount != 10 || latest.VcpuTotal != 20 || latest.RamTotalGB != 30 {
t.Fatalf("unexpected latest totals for vc-a: %#v", latest)
}
}
func TestListVcenterHourlyTotalsSince(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
base := time.Unix(1_700_000_000, 0)
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", base.AddDate(0, 0, -60), 1, 2, 3); err != nil {
t.Fatalf("failed to insert old totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", base.AddDate(0, 0, -10), 10, 20, 30); err != nil {
t.Fatalf("failed to insert recent totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-b", base.AddDate(0, 0, -5), 100, 200, 300); err != nil {
t.Fatalf("failed to insert other-vcenter totals: %v", err)
}
rows, err := ListVcenterHourlyTotalsSince(ctx, dbConn, "vc-a", base.AddDate(0, 0, -45))
if err != nil {
t.Fatalf("ListVcenterHourlyTotalsSince failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 row for vc-a since cutoff, got %d", len(rows))
}
if rows[0].SnapshotTime != base.AddDate(0, 0, -10).Unix() || rows[0].VmCount != 10 {
t.Fatalf("unexpected row returned: %#v", rows[0])
}
}
func TestInsertVcenterTotalsUpsertsHourlyAggregate(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
snapshotTime := time.Unix(1_700_000_500, 0)
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", snapshotTime, 12, 24, 48); err != nil {
t.Fatalf("InsertVcenterTotals failed: %v", err)
}
rows, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "hourly", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 hourly aggregate row, got %d", len(rows))
}
if rows[0].SnapshotTime != snapshotTime.Unix() || rows[0].VmCount != 12 || rows[0].VcpuTotal != 24 || rows[0].RamTotalGB != 48 {
t.Fatalf("unexpected hourly aggregate row: %#v", rows[0])
}
}
func TestListVcenterHourlyTotalsSinceUsesAggregateCache(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
base := time.Unix(1_700_000_000, 0)
if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", "vc-a", base.Unix(), 7, 14, 21); err != nil {
t.Fatalf("UpsertVcenterAggregateTotal failed: %v", err)
}
rows, err := ListVcenterHourlyTotalsSince(ctx, dbConn, "vc-a", base.Add(-24*time.Hour))
if err != nil {
t.Fatalf("ListVcenterHourlyTotalsSince failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 cached row, got %d", len(rows))
}
if rows[0].SnapshotTime != base.Unix() || rows[0].VmCount != 7 || rows[0].VcpuTotal != 14 || rows[0].RamTotalGB != 21 {
t.Fatalf("unexpected cached hourly row: %#v", rows[0])
}
}
func TestReplaceVcenterAggregateTotalsFromSummary(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
summaryTable := "inventory_daily_summary_20260101"
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, summaryTable)); err != nil {
t.Fatalf("failed to create summary table: %v", err)
}
insertSQL := fmt.Sprintf(`
INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB")
VALUES (?,?,?,?,?,?)
`, summaryTable)
rows := [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 2.0, 4.0},
{"vc-a", "vm-2", "2", "u2", 3.0, 5.0},
{"vc-b", "vm-3", "3", "u3", 1.0, 2.0},
}
for _, args := range rows {
if _, err := dbConn.ExecContext(ctx, insertSQL, args...); err != nil {
t.Fatalf("failed to insert summary row: %v", err)
}
}
upserted, err := ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "daily", 1_700_010_000)
if err != nil {
t.Fatalf("ReplaceVcenterAggregateTotalsFromSummary failed: %v", err)
}
if upserted != 2 {
t.Fatalf("expected 2 vcenter aggregate rows, got %d", upserted)
}
vcA, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals(vc-a) failed: %v", err)
}
if len(vcA) != 1 {
t.Fatalf("expected 1 vc-a daily row, got %d", len(vcA))
}
if vcA[0].SnapshotTime != 1_700_010_000 || vcA[0].VmCount != 2 || vcA[0].VcpuTotal != 5 || vcA[0].RamTotalGB != 9 {
t.Fatalf("unexpected vc-a daily aggregate row: %#v", vcA[0])
}
vcB, err := ListVcenterAggregateTotalsSince(ctx, dbConn, "vc-b", "daily", time.Unix(1_700_009_000, 0))
if err != nil {
t.Fatalf("ListVcenterAggregateTotalsSince(vc-b) failed: %v", err)
}
if len(vcB) != 1 || vcB[0].VmCount != 1 || vcB[0].VcpuTotal != 1 || vcB[0].RamTotalGB != 2 {
t.Fatalf("unexpected vc-b daily aggregate row: %#v", vcB)
}
}
func TestListVcenterTotalsByTypeDailyFallbackWarmsCache(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if _, err := dbConn.ExecContext(ctx, `
CREATE TABLE snapshot_registry (
snapshot_type TEXT,
table_name TEXT,
snapshot_time BIGINT
)`); err != nil {
t.Fatalf("failed to create snapshot_registry: %v", err)
}
summaryTable := "inventory_daily_summary_20260102"
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, summaryTable)); err != nil {
t.Fatalf("failed to create summary table: %v", err)
}
insertSQL := fmt.Sprintf(`
INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB")
VALUES (?,?,?,?,?,?)
`, summaryTable)
for _, args := range [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 4.0, 8.0},
{"vc-a", "vm-2", "2", "u2", 2.0, 6.0},
} {
if _, err := dbConn.ExecContext(ctx, insertSQL, args...); err != nil {
t.Fatalf("failed to insert summary row: %v", err)
}
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", summaryTable, int64(1_700_020_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row: %v", err)
}
rows, err := ListVcenterTotalsByType(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterTotalsByType failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 daily row, got %d", len(rows))
}
if rows[0].SnapshotTime != 1_700_020_000 || rows[0].VmCount != 2 || rows[0].VcpuTotal != 6 || rows[0].RamTotalGB != 14 {
t.Fatalf("unexpected daily totals row: %#v", rows[0])
}
cached, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(cached) != 1 || cached[0].SnapshotTime != 1_700_020_000 || cached[0].VmCount != 2 {
t.Fatalf("expected warmed daily cache row, got %#v", cached)
}
}
func TestSyncVcenterAggregateTotalsFromRegistry(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if _, err := dbConn.ExecContext(ctx, `
CREATE TABLE snapshot_registry (
snapshot_type TEXT,
table_name TEXT,
snapshot_time BIGINT
)`); err != nil {
t.Fatalf("failed to create snapshot_registry: %v", err)
}
table1 := "inventory_daily_summary_20260103"
table2 := "inventory_daily_summary_20260104"
for _, table := range []string{table1, table2} {
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, table)); err != nil {
t.Fatalf("failed to create summary table %s: %v", table, err)
}
}
insert1 := fmt.Sprintf(`INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB") VALUES (?,?,?,?,?,?)`, table1)
insert2 := fmt.Sprintf(`INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB") VALUES (?,?,?,?,?,?)`, table2)
for _, args := range [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 2.0, 4.0},
{"vc-b", "vm-2", "2", "u2", 3.0, 5.0},
} {
if _, err := dbConn.ExecContext(ctx, insert1, args...); err != nil {
t.Fatalf("failed to insert row into %s: %v", table1, err)
}
}
if _, err := dbConn.ExecContext(ctx, insert2, "vc-a", "vm-3", "3", "u3", 4.0, 6.0); err != nil {
t.Fatalf("failed to insert row into %s: %v", table2, err)
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", table1, int64(1_700_030_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row for table1: %v", err)
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", table2, int64(1_700_040_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row for table2: %v", err)
}
snapshotsRefreshed, rowsUpserted, err := SyncVcenterAggregateTotalsFromRegistry(ctx, dbConn, "daily")
if err != nil {
t.Fatalf("SyncVcenterAggregateTotalsFromRegistry failed: %v", err)
}
if snapshotsRefreshed != 2 {
t.Fatalf("expected 2 snapshots refreshed, got %d", snapshotsRefreshed)
}
if rowsUpserted != 3 {
t.Fatalf("expected 3 rows upserted, got %d", rowsUpserted)
}
rows, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(rows) != 2 {
t.Fatalf("expected 2 daily rows for vc-a, got %d", len(rows))
}
if rows[0].SnapshotTime != 1_700_040_000 || rows[0].VmCount != 1 || rows[0].VcpuTotal != 4 || rows[0].RamTotalGB != 6 {
t.Fatalf("unexpected latest vc-a daily row: %#v", rows[0])
}
if rows[1].SnapshotTime != 1_700_030_000 || rows[1].VmCount != 1 || rows[1].VcpuTotal != 2 || rows[1].RamTotalGB != 4 {
t.Fatalf("unexpected older vc-a daily row: %#v", rows[1])
}
}