Files
vctp2/db/helpers_cache_and_index_test.go
Nathan Coad 5736dc6929
Some checks failed
continuous-integration/drone/push Build is failing
avoid vcenter totals pages scanning whole database
2026-02-09 13:44:43 +11:00

478 lines
16 KiB
Go

package db
import (
"context"
"database/sql"
"fmt"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "modernc.org/sqlite"
)
func newTestSQLiteDB(t *testing.T) *sqlx.DB {
t.Helper()
dbConn, err := sqlx.Open("sqlite", ":memory:")
if err != nil {
t.Fatalf("failed to open sqlite test db: %v", err)
}
t.Cleanup(func() {
_ = dbConn.Close()
})
return dbConn
}
func indexExists(t *testing.T, dbConn *sqlx.DB, name string) bool {
t.Helper()
var count int
if err := dbConn.Get(&count, `SELECT COUNT(1) FROM sqlite_master WHERE type='index' AND name=?`, name); err != nil {
t.Fatalf("failed to query index %s: %v", name, err)
}
return count > 0
}
func TestCleanupHourlySnapshotIndexesOlderThan(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
oldTable := "inventory_hourly_1700000000"
newTable := "inventory_hourly_1800000000"
for _, table := range []string{oldTable, newTable} {
if err := EnsureSnapshotTable(ctx, dbConn, table); err != nil {
t.Fatalf("failed to create snapshot table %s: %v", table, err)
}
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_snapshottime_idx ON %s ("SnapshotTime")`, table, table)); err != nil {
t.Fatalf("failed to create snapshottime index for %s: %v", table, err)
}
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, table, table)); err != nil {
t.Fatalf("failed to create resourcepool index for %s: %v", table, err)
}
}
cutoff := time.Unix(1750000000, 0)
dropped, err := CleanupHourlySnapshotIndexesOlderThan(ctx, dbConn, cutoff)
if err != nil {
t.Fatalf("cleanup failed: %v", err)
}
if dropped != 3 {
t.Fatalf("expected 3 old indexes dropped, got %d", dropped)
}
oldIndexes := []string{
oldTable + "_vm_vcenter_idx",
oldTable + "_snapshottime_idx",
oldTable + "_resourcepool_idx",
}
for _, idx := range oldIndexes {
if indexExists(t, dbConn, idx) {
t.Fatalf("expected old index %s to be removed", idx)
}
}
newIndexes := []string{
newTable + "_vm_vcenter_idx",
newTable + "_snapshottime_idx",
newTable + "_resourcepool_idx",
}
for _, idx := range newIndexes {
if !indexExists(t, dbConn, idx) {
t.Fatalf("expected recent index %s to remain", idx)
}
}
}
func TestFetchVmTraceAndLifecycleUseCacheTables(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if err := EnsureVmHourlyStats(ctx, dbConn); err != nil {
t.Fatalf("failed to ensure vm_hourly_stats: %v", err)
}
if err := EnsureVmLifecycleCache(ctx, dbConn); err != nil {
t.Fatalf("failed to ensure vm_lifecycle_cache: %v", err)
}
insertSQL := `
INSERT INTO vm_hourly_stats (
"SnapshotTime","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","ResourcePool",
"Datacenter","Cluster","Folder","ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder"
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
`
rows := [][]interface{}{
{int64(1000), "vc-a", "vm-1", "uuid-1", "demo-vm", int64(900), int64(0), "Tin", "dc", "cluster", "folder", 100.0, int64(2), int64(4), "FALSE", "TRUE", "FALSE"},
{int64(2000), "vc-a", "vm-1", "uuid-1", "demo-vm", int64(900), int64(0), "Gold", "dc", "cluster", "folder", 150.0, int64(4), int64(8), "FALSE", "TRUE", "FALSE"},
}
for _, args := range rows {
if _, err := dbConn.ExecContext(ctx, insertSQL, args...); err != nil {
t.Fatalf("failed to insert hourly cache row: %v", err)
}
}
if err := UpsertVmLifecycleCache(ctx, dbConn, "vc-a", "vm-1", "uuid-1", "demo-vm", "cluster", time.Unix(1000, 0), sql.NullInt64{Int64: 900, Valid: true}); err != nil {
t.Fatalf("failed to upsert lifecycle cache: %v", err)
}
if err := MarkVmDeletedWithDetails(ctx, dbConn, "vc-a", "vm-1", "uuid-1", "demo-vm", "cluster", 2500); err != nil {
t.Fatalf("failed to mark vm deleted: %v", err)
}
traceRows, err := FetchVmTrace(ctx, dbConn, "vm-1", "", "")
if err != nil {
t.Fatalf("FetchVmTrace failed: %v", err)
}
if len(traceRows) != 2 {
t.Fatalf("expected 2 trace rows, got %d", len(traceRows))
}
if traceRows[0].SnapshotTime != 1000 || traceRows[1].SnapshotTime != 2000 {
t.Fatalf("trace rows are not sorted by snapshot time: %#v", traceRows)
}
lifecycle, err := FetchVmLifecycle(ctx, dbConn, "vm-1", "", "")
if err != nil {
t.Fatalf("FetchVmLifecycle failed: %v", err)
}
if lifecycle.FirstSeen != 900 {
t.Fatalf("expected FirstSeen=900 (earliest known from lifecycle cache), got %d", lifecycle.FirstSeen)
}
if lifecycle.LastSeen != 2000 {
t.Fatalf("expected LastSeen=2000, got %d", lifecycle.LastSeen)
}
if lifecycle.CreationTime != 900 || lifecycle.CreationApprox {
t.Fatalf("expected exact CreationTime=900, got time=%d approx=%v", lifecycle.CreationTime, lifecycle.CreationApprox)
}
if lifecycle.DeletionTime != 2500 {
t.Fatalf("expected DeletionTime=2500 from lifecycle cache, got %d", lifecycle.DeletionTime)
}
}
func TestParseHourlySnapshotUnix(t *testing.T) {
cases := []struct {
table string
ok bool
val int64
}{
{table: "inventory_hourly_1700000000", ok: true, val: 1700000000},
{table: "inventory_hourly_bad", ok: false, val: 0},
{table: "inventory_daily_summary_20260101", ok: false, val: 0},
}
for _, tc := range cases {
got, ok := parseHourlySnapshotUnix(tc.table)
if ok != tc.ok || got != tc.val {
t.Fatalf("parseHourlySnapshotUnix(%q) = (%d,%v), expected (%d,%v)", tc.table, got, ok, tc.val, tc.ok)
}
}
}
func TestVcenterLatestTotalsAndListFallback(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if err := EnsureVcenterLatestTotalsTable(ctx, dbConn); err != nil {
t.Fatalf("failed to ensure vcenter_latest_totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", time.Unix(200, 0), 10, 20, 30); err != nil {
t.Fatalf("failed to insert totals for vc-a: %v", err)
}
// Older snapshot should not replace latest totals.
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", time.Unix(100, 0), 1, 2, 3); err != nil {
t.Fatalf("failed to insert older totals for vc-a: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-b", time.Unix(300, 0), 11, 21, 31); err != nil {
t.Fatalf("failed to insert totals for vc-b: %v", err)
}
vcenters, err := ListVcenters(ctx, dbConn)
if err != nil {
t.Fatalf("ListVcenters failed: %v", err)
}
if len(vcenters) != 2 || vcenters[0] != "vc-a" || vcenters[1] != "vc-b" {
t.Fatalf("unexpected vcenter list: %#v", vcenters)
}
var latest struct {
SnapshotTime int64 `db:"SnapshotTime"`
VmCount int64 `db:"VmCount"`
VcpuTotal int64 `db:"VcpuTotal"`
RamTotalGB int64 `db:"RamTotalGB"`
}
if err := dbConn.GetContext(ctx, &latest, `
SELECT "SnapshotTime","VmCount","VcpuTotal","RamTotalGB"
FROM vcenter_latest_totals
WHERE "Vcenter" = ?
`, "vc-a"); err != nil {
t.Fatalf("failed to query latest totals for vc-a: %v", err)
}
if latest.SnapshotTime != 200 || latest.VmCount != 10 || latest.VcpuTotal != 20 || latest.RamTotalGB != 30 {
t.Fatalf("unexpected latest totals for vc-a: %#v", latest)
}
}
func TestListVcenterHourlyTotalsSince(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
base := time.Unix(1_700_000_000, 0)
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", base.AddDate(0, 0, -60), 1, 2, 3); err != nil {
t.Fatalf("failed to insert old totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", base.AddDate(0, 0, -10), 10, 20, 30); err != nil {
t.Fatalf("failed to insert recent totals: %v", err)
}
if err := InsertVcenterTotals(ctx, dbConn, "vc-b", base.AddDate(0, 0, -5), 100, 200, 300); err != nil {
t.Fatalf("failed to insert other-vcenter totals: %v", err)
}
rows, err := ListVcenterHourlyTotalsSince(ctx, dbConn, "vc-a", base.AddDate(0, 0, -45))
if err != nil {
t.Fatalf("ListVcenterHourlyTotalsSince failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 row for vc-a since cutoff, got %d", len(rows))
}
if rows[0].SnapshotTime != base.AddDate(0, 0, -10).Unix() || rows[0].VmCount != 10 {
t.Fatalf("unexpected row returned: %#v", rows[0])
}
}
func TestInsertVcenterTotalsUpsertsHourlyAggregate(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
snapshotTime := time.Unix(1_700_000_500, 0)
if err := InsertVcenterTotals(ctx, dbConn, "vc-a", snapshotTime, 12, 24, 48); err != nil {
t.Fatalf("InsertVcenterTotals failed: %v", err)
}
rows, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "hourly", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 hourly aggregate row, got %d", len(rows))
}
if rows[0].SnapshotTime != snapshotTime.Unix() || rows[0].VmCount != 12 || rows[0].VcpuTotal != 24 || rows[0].RamTotalGB != 48 {
t.Fatalf("unexpected hourly aggregate row: %#v", rows[0])
}
}
func TestListVcenterHourlyTotalsSinceUsesAggregateCache(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
base := time.Unix(1_700_000_000, 0)
if err := UpsertVcenterAggregateTotal(ctx, dbConn, "hourly", "vc-a", base.Unix(), 7, 14, 21); err != nil {
t.Fatalf("UpsertVcenterAggregateTotal failed: %v", err)
}
rows, err := ListVcenterHourlyTotalsSince(ctx, dbConn, "vc-a", base.Add(-24*time.Hour))
if err != nil {
t.Fatalf("ListVcenterHourlyTotalsSince failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 cached row, got %d", len(rows))
}
if rows[0].SnapshotTime != base.Unix() || rows[0].VmCount != 7 || rows[0].VcpuTotal != 14 || rows[0].RamTotalGB != 21 {
t.Fatalf("unexpected cached hourly row: %#v", rows[0])
}
}
func TestReplaceVcenterAggregateTotalsFromSummary(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
summaryTable := "inventory_daily_summary_20260101"
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, summaryTable)); err != nil {
t.Fatalf("failed to create summary table: %v", err)
}
insertSQL := fmt.Sprintf(`
INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB")
VALUES (?,?,?,?,?,?)
`, summaryTable)
rows := [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 2.0, 4.0},
{"vc-a", "vm-2", "2", "u2", 3.0, 5.0},
{"vc-b", "vm-3", "3", "u3", 1.0, 2.0},
}
for _, args := range rows {
if _, err := dbConn.ExecContext(ctx, insertSQL, args...); err != nil {
t.Fatalf("failed to insert summary row: %v", err)
}
}
upserted, err := ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "daily", 1_700_010_000)
if err != nil {
t.Fatalf("ReplaceVcenterAggregateTotalsFromSummary failed: %v", err)
}
if upserted != 2 {
t.Fatalf("expected 2 vcenter aggregate rows, got %d", upserted)
}
vcA, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals(vc-a) failed: %v", err)
}
if len(vcA) != 1 {
t.Fatalf("expected 1 vc-a daily row, got %d", len(vcA))
}
if vcA[0].SnapshotTime != 1_700_010_000 || vcA[0].VmCount != 2 || vcA[0].VcpuTotal != 5 || vcA[0].RamTotalGB != 9 {
t.Fatalf("unexpected vc-a daily aggregate row: %#v", vcA[0])
}
vcB, err := ListVcenterAggregateTotalsSince(ctx, dbConn, "vc-b", "daily", time.Unix(1_700_009_000, 0))
if err != nil {
t.Fatalf("ListVcenterAggregateTotalsSince(vc-b) failed: %v", err)
}
if len(vcB) != 1 || vcB[0].VmCount != 1 || vcB[0].VcpuTotal != 1 || vcB[0].RamTotalGB != 2 {
t.Fatalf("unexpected vc-b daily aggregate row: %#v", vcB)
}
}
func TestListVcenterTotalsByTypeDailyFallbackWarmsCache(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if _, err := dbConn.ExecContext(ctx, `
CREATE TABLE snapshot_registry (
snapshot_type TEXT,
table_name TEXT,
snapshot_time BIGINT
)`); err != nil {
t.Fatalf("failed to create snapshot_registry: %v", err)
}
summaryTable := "inventory_daily_summary_20260102"
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, summaryTable)); err != nil {
t.Fatalf("failed to create summary table: %v", err)
}
insertSQL := fmt.Sprintf(`
INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB")
VALUES (?,?,?,?,?,?)
`, summaryTable)
for _, args := range [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 4.0, 8.0},
{"vc-a", "vm-2", "2", "u2", 2.0, 6.0},
} {
if _, err := dbConn.ExecContext(ctx, insertSQL, args...); err != nil {
t.Fatalf("failed to insert summary row: %v", err)
}
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", summaryTable, int64(1_700_020_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row: %v", err)
}
rows, err := ListVcenterTotalsByType(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterTotalsByType failed: %v", err)
}
if len(rows) != 1 {
t.Fatalf("expected 1 daily row, got %d", len(rows))
}
if rows[0].SnapshotTime != 1_700_020_000 || rows[0].VmCount != 2 || rows[0].VcpuTotal != 6 || rows[0].RamTotalGB != 14 {
t.Fatalf("unexpected daily totals row: %#v", rows[0])
}
cached, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(cached) != 1 || cached[0].SnapshotTime != 1_700_020_000 || cached[0].VmCount != 2 {
t.Fatalf("expected warmed daily cache row, got %#v", cached)
}
}
func TestSyncVcenterAggregateTotalsFromRegistry(t *testing.T) {
ctx := context.Background()
dbConn := newTestSQLiteDB(t)
if _, err := dbConn.ExecContext(ctx, `
CREATE TABLE snapshot_registry (
snapshot_type TEXT,
table_name TEXT,
snapshot_time BIGINT
)`); err != nil {
t.Fatalf("failed to create snapshot_registry: %v", err)
}
table1 := "inventory_daily_summary_20260103"
table2 := "inventory_daily_summary_20260104"
for _, table := range []string{table1, table2} {
if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(`
CREATE TABLE %s (
"Vcenter" TEXT NOT NULL,
"Name" TEXT,
"VmId" TEXT,
"VmUuid" TEXT,
"AvgVcpuCount" REAL,
"VcpuCount" BIGINT,
"AvgRamGB" REAL,
"RamGB" BIGINT
)`, table)); err != nil {
t.Fatalf("failed to create summary table %s: %v", table, err)
}
}
insert1 := fmt.Sprintf(`INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB") VALUES (?,?,?,?,?,?)`, table1)
insert2 := fmt.Sprintf(`INSERT INTO %s ("Vcenter","Name","VmId","VmUuid","AvgVcpuCount","AvgRamGB") VALUES (?,?,?,?,?,?)`, table2)
for _, args := range [][]interface{}{
{"vc-a", "vm-1", "1", "u1", 2.0, 4.0},
{"vc-b", "vm-2", "2", "u2", 3.0, 5.0},
} {
if _, err := dbConn.ExecContext(ctx, insert1, args...); err != nil {
t.Fatalf("failed to insert row into %s: %v", table1, err)
}
}
if _, err := dbConn.ExecContext(ctx, insert2, "vc-a", "vm-3", "3", "u3", 4.0, 6.0); err != nil {
t.Fatalf("failed to insert row into %s: %v", table2, err)
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", table1, int64(1_700_030_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row for table1: %v", err)
}
if _, err := dbConn.ExecContext(ctx, `INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) VALUES (?,?,?)`, "daily", table2, int64(1_700_040_000)); err != nil {
t.Fatalf("failed to insert snapshot_registry row for table2: %v", err)
}
snapshotsRefreshed, rowsUpserted, err := SyncVcenterAggregateTotalsFromRegistry(ctx, dbConn, "daily")
if err != nil {
t.Fatalf("SyncVcenterAggregateTotalsFromRegistry failed: %v", err)
}
if snapshotsRefreshed != 2 {
t.Fatalf("expected 2 snapshots refreshed, got %d", snapshotsRefreshed)
}
if rowsUpserted != 3 {
t.Fatalf("expected 3 rows upserted, got %d", rowsUpserted)
}
rows, err := ListVcenterAggregateTotals(ctx, dbConn, "vc-a", "daily", 10)
if err != nil {
t.Fatalf("ListVcenterAggregateTotals failed: %v", err)
}
if len(rows) != 2 {
t.Fatalf("expected 2 daily rows for vc-a, got %d", len(rows))
}
if rows[0].SnapshotTime != 1_700_040_000 || rows[0].VmCount != 1 || rows[0].VcpuTotal != 4 || rows[0].RamTotalGB != 6 {
t.Fatalf("unexpected latest vc-a daily row: %#v", rows[0])
}
if rows[1].SnapshotTime != 1_700_030_000 || rows[1].VmCount != 1 || rows[1].VcpuTotal != 2 || rows[1].RamTotalGB != 4 {
t.Fatalf("unexpected older vc-a daily row: %#v", rows[1])
}
}