From 916b0b5054aa90e049cc02f71ddb37938e61e205 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Mon, 20 Apr 2026 18:38:12 +1000 Subject: [PATCH] more tests --- .../tasks/aggregation_integration_test.go | 511 ++++++++++++++++++ internal/tasks/aggregation_parity_test.go | 10 +- .../tasks/compatibility_integration_test.go | 212 ++++++++ internal/tasks/monthlyAggregate.go | 20 + plan.md | 6 +- .../snapshot_repair_integration_test.go | 181 +++++++ 6 files changed, 933 insertions(+), 7 deletions(-) create mode 100644 internal/tasks/aggregation_integration_test.go create mode 100644 internal/tasks/compatibility_integration_test.go create mode 100644 server/handler/snapshot_repair_integration_test.go diff --git a/internal/tasks/aggregation_integration_test.go b/internal/tasks/aggregation_integration_test.go new file mode 100644 index 0000000..2a7bbf6 --- /dev/null +++ b/internal/tasks/aggregation_integration_test.go @@ -0,0 +1,511 @@ +package tasks + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + "vctp/db" + "vctp/internal/settings" + + "github.com/jmoiron/sqlx" +) + +func TestCanonicalDailyFlow_WritesRollupAndTotalsCache(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTask(dbConn) + + if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_hourly_stats: %v", err) + } + + dayStart := time.Date(2026, time.March, 10, 0, 0, 0, 0, time.UTC) + dayEnd := dayStart.AddDate(0, 0, 1) + t1 := dayStart.Add(1 * time.Hour).Unix() + t2 := dayStart.Add(2 * time.Hour).Unix() + t3 := dayStart.Add(3 * time.Hour).Unix() + + seeds := []hourlySeedRow{ + {SnapshotTime: t1, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 100, VcpuCount: 2, RamGB: 8, CreationTime: dayStart.Add(-1 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", ResourcePool: "Gold", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 120, VcpuCount: 4, RamGB: 8, CreationTime: dayStart.Add(-1 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-a2", Vcenter: "vc-a", VmID: "vm-a2", VmUUID: "uuid-a2", ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 40, VcpuCount: 1, RamGB: 4, CreationTime: dayStart.Add(-2 * time.Hour).Unix(), DeletionTime: dayStart.Add(4 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t1, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t3, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + } + for _, seed := range seeds { + if err := insertHourlyCacheSeedRow(ctx, dbConn, seed); err != nil { + t.Fatalf("failed to insert hourly seed row: %v", err) + } + } + + aggMap, snapTimes, err := task.scanHourlyCache(ctx, dayStart, dayEnd) + if err != nil { + t.Fatalf("scanHourlyCache failed: %v", err) + } + if len(aggMap) != 3 { + t.Fatalf("unexpected daily agg key count: got %d want %d", len(aggMap), 3) + } + if len(snapTimes) != 3 { + t.Fatalf("unexpected snapshot time count: got %d want %d", len(snapTimes), 3) + } + totalSamplesByVcenter := sampleCountsByVcenter(aggMap) + if totalSamplesByVcenter["vc-a"] != 2 || totalSamplesByVcenter["vc-b"] != 3 { + t.Fatalf("unexpected per-vcenter sample counts: %#v", totalSamplesByVcenter) + } + + summaryTable, err := db.SafeTableName("test_daily_canonical_integration_summary") + if err != nil { + t.Fatalf("failed to build summary table name: %v", err) + } + if err := db.EnsureSummaryTable(ctx, dbConn, summaryTable); err != nil { + t.Fatalf("failed to ensure summary table: %v", err) + } + if err := task.insertDailyAggregates(ctx, summaryTable, aggMap, len(snapTimes), totalSamplesByVcenter); err != nil { + t.Fatalf("insertDailyAggregates failed: %v", err) + } + if err := task.persistDailyRollup(ctx, dayStart.Unix(), aggMap, len(snapTimes), totalSamplesByVcenter); err != nil { + t.Fatalf("persistDailyRollup failed: %v", err) + } + + rollupAgg, err := task.scanDailyRollup(ctx, dayStart, dayEnd) + if err != nil { + t.Fatalf("scanDailyRollup failed: %v", err) + } + if len(rollupAgg) != len(aggMap) { + t.Fatalf("unexpected rollup agg key count: got %d want %d", len(rollupAgg), len(aggMap)) + } + + refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "daily", dayStart.Unix()) + if err != nil { + t.Fatalf("ReplaceVcenterAggregateTotalsFromSummary(daily) failed: %v", err) + } + if refreshed != 2 { + t.Fatalf("unexpected daily refreshed vcenter rows: got %d want %d", refreshed, 2) + } + + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "daily", dayStart.Unix()) + assertRollupTotalSamplesForVcenter(t, ctx, dbConn, dayStart.Unix(), "vc-a", 2) + assertRollupTotalSamplesForVcenter(t, ctx, dbConn, dayStart.Unix(), "vc-b", 3) +} + +func TestCanonicalMonthlyFlow_WritesSummaryAndTotalsCache(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTask(dbConn) + + if err := db.EnsureVmDailyRollup(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_daily_rollup: %v", err) + } + + monthStart := time.Date(2026, time.April, 1, 0, 0, 0, 0, time.UTC) + monthEnd := monthStart.AddDate(0, 1, 0) + day1 := monthStart.AddDate(0, 0, 5).Unix() + day2 := monthStart.AddDate(0, 0, 6).Unix() + + rollupSeeds := []dailySeedRow{ + { + SnapshotTime: day1, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", + ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", + ProvisionedDisk: 120, VcpuCount: 4, RamGB: 8, CreationTime: monthStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 6, SumRam: 12, SumDisk: 240, BronzeHits: 2, + }, + { + SnapshotTime: day2, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", + ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", + ProvisionedDisk: 110, VcpuCount: 2, RamGB: 8, CreationTime: monthStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 4, SumRam: 16, SumDisk: 220, TinHits: 2, + }, + { + SnapshotTime: day1, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", + ResourcePool: "Gold", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", + ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: monthStart.Add(-10 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 16, SumRam: 64, SumDisk: 400, GoldHits: 2, + }, + { + SnapshotTime: day2, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", + ResourcePool: "Gold", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", + ProvisionedDisk: 210, VcpuCount: 8, RamGB: 32, CreationTime: monthStart.Add(-10 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 16, SumRam: 64, SumDisk: 420, GoldHits: 2, + }, + } + for _, seed := range rollupSeeds { + if err := insertDailyRollupSeedRow(ctx, dbConn, seed); err != nil { + t.Fatalf("failed to insert daily rollup seed row: %v", err) + } + } + + aggMap, err := task.scanDailyRollup(ctx, monthStart, monthEnd) + if err != nil { + t.Fatalf("scanDailyRollup failed: %v", err) + } + if len(aggMap) != 2 { + t.Fatalf("unexpected monthly agg key count: got %d want %d", len(aggMap), 2) + } + + summaryTable, err := db.SafeTableName("test_monthly_canonical_integration_summary") + if err != nil { + t.Fatalf("failed to build monthly summary table name: %v", err) + } + if err := db.EnsureSummaryTable(ctx, dbConn, summaryTable); err != nil { + t.Fatalf("failed to ensure monthly summary table: %v", err) + } + if err := task.insertMonthlyAggregates(ctx, summaryTable, aggMap); err != nil { + t.Fatalf("insertMonthlyAggregates failed: %v", err) + } + + refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "monthly", monthStart.Unix()) + if err != nil { + t.Fatalf("ReplaceVcenterAggregateTotalsFromSummary(monthly) failed: %v", err) + } + if refreshed != 2 { + t.Fatalf("unexpected monthly refreshed vcenter rows: got %d want %d", refreshed, 2) + } + + monthlyRows, err := loadMonthlySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load monthly summary rows: %v", err) + } + if len(monthlyRows) != 2 { + t.Fatalf("unexpected monthly summary row count: got %d want %d", len(monthlyRows), 2) + } + + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "monthly", monthStart.Unix()) +} + +func TestScheduledCanonicalDailyTaskFlow_WritesSummaryRollupRegistryAndTotalsCache(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTaskForAggregateFlow(t, dbConn) + + if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_hourly_stats: %v", err) + } + + dayStart := time.Date(2026, time.March, 12, 0, 0, 0, 0, time.UTC) + t1 := dayStart.Add(1 * time.Hour).Unix() + t2 := dayStart.Add(2 * time.Hour).Unix() + t3 := dayStart.Add(3 * time.Hour).Unix() + seeds := []hourlySeedRow{ + {SnapshotTime: t1, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 100, VcpuCount: 2, RamGB: 8, CreationTime: dayStart.Add(-1 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", ResourcePool: "Gold", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 120, VcpuCount: 4, RamGB: 8, CreationTime: dayStart.Add(-1 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-a2", Vcenter: "vc-a", VmID: "vm-a2", VmUUID: "uuid-a2", ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 40, VcpuCount: 1, RamGB: 4, CreationTime: dayStart.Add(-2 * time.Hour).Unix(), DeletionTime: dayStart.Add(4 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t1, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t3, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", ResourcePool: "Silver", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: dayStart.Add(-3 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + } + for _, seed := range seeds { + if err := insertHourlyCacheSeedRow(ctx, dbConn, seed); err != nil { + t.Fatalf("failed to insert hourly seed row: %v", err) + } + } + + if err := task.aggregateDailySummaryWithMode(ctx, dayStart, true, true); err != nil { + t.Fatalf("aggregateDailySummaryWithMode failed: %v", err) + } + + summaryTable, err := dailySummaryTableName(dayStart) + if err != nil { + t.Fatalf("failed to build summary table name: %v", err) + } + rows, err := loadDailySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load daily summary rows: %v", err) + } + if len(rows) != 3 { + t.Fatalf("unexpected daily summary row count: got %d want %d", len(rows), 3) + } + + assertSnapshotRegistryRow(t, ctx, dbConn, "daily", summaryTable, dayStart.Unix(), int64(len(rows))) + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "daily", dayStart.Unix()) + assertRollupTotalSamplesForVcenter(t, ctx, dbConn, dayStart.Unix(), "vc-a", 2) + assertRollupTotalSamplesForVcenter(t, ctx, dbConn, dayStart.Unix(), "vc-b", 3) +} + +func TestScheduledCanonicalMonthlyTaskFlow_WritesSummaryRegistryAndTotalsCache(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTaskForAggregateFlow(t, dbConn) + + if err := db.EnsureVmDailyRollup(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_daily_rollup: %v", err) + } + + targetMonth := time.Date(2026, time.April, 20, 0, 0, 0, 0, time.UTC) + monthStart := time.Date(targetMonth.Year(), targetMonth.Month(), 1, 0, 0, 0, 0, targetMonth.Location()) + day1 := monthStart.AddDate(0, 0, 5).Unix() + day2 := monthStart.AddDate(0, 0, 6).Unix() + rollupSeeds := []dailySeedRow{ + { + SnapshotTime: day1, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", + ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", + ProvisionedDisk: 120, VcpuCount: 4, RamGB: 8, CreationTime: monthStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 6, SumRam: 12, SumDisk: 240, BronzeHits: 2, + }, + { + SnapshotTime: day2, Name: "vm-a1", Vcenter: "vc-a", VmID: "vm-a1", VmUUID: "uuid-a1", + ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", + ProvisionedDisk: 110, VcpuCount: 2, RamGB: 8, CreationTime: monthStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 4, SumRam: 16, SumDisk: 220, TinHits: 2, + }, + { + SnapshotTime: day1, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", + ResourcePool: "Gold", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", + ProvisionedDisk: 200, VcpuCount: 8, RamGB: 32, CreationTime: monthStart.Add(-10 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 16, SumRam: 64, SumDisk: 400, GoldHits: 2, + }, + { + SnapshotTime: day2, Name: "vm-b1", Vcenter: "vc-b", VmID: "vm-b1", VmUUID: "uuid-b1", + ResourcePool: "Gold", Datacenter: "dc-b", Cluster: "cluster-b", Folder: "/prod", + ProvisionedDisk: 210, VcpuCount: 8, RamGB: 32, CreationTime: monthStart.Add(-10 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE", + SamplesPresent: 2, TotalSamples: 2, SumVcpu: 16, SumRam: 64, SumDisk: 420, GoldHits: 2, + }, + } + for _, seed := range rollupSeeds { + if err := insertDailyRollupSeedRow(ctx, dbConn, seed); err != nil { + t.Fatalf("failed to insert daily rollup seed row: %v", err) + } + } + + if err := task.aggregateMonthlySummaryWithMode(ctx, targetMonth, true, true); err != nil { + t.Fatalf("aggregateMonthlySummaryWithMode failed: %v", err) + } + + summaryTable, err := monthlySummaryTableName(targetMonth) + if err != nil { + t.Fatalf("failed to build monthly summary table name: %v", err) + } + rows, err := loadMonthlySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load monthly summary rows: %v", err) + } + if len(rows) != 2 { + t.Fatalf("unexpected monthly summary row count: got %d want %d", len(rows), 2) + } + + assertSnapshotRegistryRow(t, ctx, dbConn, "monthly", summaryTable, monthStart.Unix(), int64(len(rows))) + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "monthly", monthStart.Unix()) +} + +func TestScheduledCanonicalDailyTaskFlow_LifecycleEdgeCases(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTaskForAggregateFlow(t, dbConn) + + if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_hourly_stats: %v", err) + } + if err := db.EnsureVmLifecycleCache(ctx, dbConn); err != nil { + t.Fatalf("failed to ensure vm_lifecycle_cache: %v", err) + } + + dayStart := time.Date(2026, time.March, 13, 0, 0, 0, 0, time.UTC) + dayEnd := dayStart.AddDate(0, 0, 1) + t1 := dayStart.Add(1 * time.Hour).Unix() + t2 := dayStart.Add(2 * time.Hour).Unix() + t3 := dayStart.Add(3 * time.Hour).Unix() + + seeds := []hourlySeedRow{ + // Deleted VM: appears only once; deletion should be inferred at first missing snapshot (t2). + {SnapshotTime: t1, Name: "vm-gone", Vcenter: "vc-a", VmID: "vm-g", VmUUID: "uuid-g", ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 80, VcpuCount: 4, RamGB: 16, CreationTime: dayStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + // Resource-change VM: verify last pool + averaged CPU/RAM/disk + pool mix percentages. + {SnapshotTime: t1, Name: "vm-change", Vcenter: "vc-a", VmID: "vm-c", VmUUID: "uuid-c", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 100, VcpuCount: 2, RamGB: 8, CreationTime: dayStart.Add(-48 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-change", Vcenter: "vc-a", VmID: "vm-c", VmUUID: "uuid-c", ResourcePool: "Silver", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 120, VcpuCount: 4, RamGB: 16, CreationTime: dayStart.Add(-48 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t3, Name: "vm-change", Vcenter: "vc-a", VmID: "vm-c", VmUUID: "uuid-c", ResourcePool: "Gold", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 140, VcpuCount: 6, RamGB: 24, CreationTime: dayStart.Add(-48 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + // Missing-creation VM: snapshot rows lack CreationTime; lifecycle cache should backfill FirstSeen (t2). + {SnapshotTime: t2, Name: "vm-partial", Vcenter: "vc-a", VmID: "vm-p", VmUUID: "uuid-p", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 60, VcpuCount: 2, RamGB: 8, CreationTime: 0, IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t3, Name: "vm-partial", Vcenter: "vc-a", VmID: "vm-p", VmUUID: "uuid-p", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 60, VcpuCount: 2, RamGB: 8, CreationTime: 0, IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + } + for _, seed := range seeds { + if err := insertHourlyCacheSeedRow(ctx, dbConn, seed); err != nil { + t.Fatalf("failed to insert hourly edge-case seed row: %v", err) + } + } + + if err := db.UpsertVmLifecycleCache(ctx, dbConn, "vc-a", "vm-p", "uuid-p", "vm-partial", "cluster-a", time.Unix(t2, 0), sql.NullInt64{}); err != nil { + t.Fatalf("failed to upsert lifecycle cache for vm-partial: %v", err) + } + + if err := task.aggregateDailySummaryWithMode(ctx, dayStart, true, true); err != nil { + t.Fatalf("aggregateDailySummaryWithMode failed: %v", err) + } + + summaryTable, err := dailySummaryTableName(dayStart) + if err != nil { + t.Fatalf("failed to build summary table name: %v", err) + } + rows, err := loadDailySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load daily summary rows: %v", err) + } + if len(rows) != 3 { + t.Fatalf("unexpected daily summary row count: got %d want %d", len(rows), 3) + } + byKey := mapRowsByKeyDaily(rows) + + partial := byKey["vc-a|vm-p|uuid-p|vm-partial"] + if partial.CreationTime != t2 { + t.Fatalf("expected vm-partial creation to be backfilled from lifecycle FirstSeen: got %d want %d", partial.CreationTime, t2) + } + wantPartialPresence := float64(dayEnd.Unix()-t2) / float64(dayEnd.Unix()-dayStart.Unix()) + if !approxEqual(partial.AvgIsPresent, wantPartialPresence, 1e-9) { + t.Fatalf("unexpected vm-partial AvgIsPresent after lifecycle creation backfill: got %.12f want %.12f", partial.AvgIsPresent, wantPartialPresence) + } + + gone := byKey["vc-a|vm-g|uuid-g|vm-gone"] + if gone.DeletionTime != t2 { + t.Fatalf("expected vm-gone deletion to be inferred from consecutive misses: got %d want %d", gone.DeletionTime, t2) + } + wantGonePresence := float64(t2-dayStart.Unix()) / float64(dayEnd.Unix()-dayStart.Unix()) + if !approxEqual(gone.AvgIsPresent, wantGonePresence, 1e-9) { + t.Fatalf("unexpected vm-gone AvgIsPresent after inferred deletion: got %.12f want %.12f", gone.AvgIsPresent, wantGonePresence) + } + + change := byKey["vc-a|vm-c|uuid-c|vm-change"] + if change.ResourcePool != "Gold" { + t.Fatalf("unexpected vm-change ResourcePool: got %q want %q", change.ResourcePool, "Gold") + } + if !approxEqual(change.AvgVcpuCount, 4.0, 1e-9) { + t.Fatalf("unexpected vm-change AvgVcpuCount: got %.12f want %.12f", change.AvgVcpuCount, 4.0) + } + if !approxEqual(change.AvgRamGB, 16.0, 1e-9) { + t.Fatalf("unexpected vm-change AvgRamGB: got %.12f want %.12f", change.AvgRamGB, 16.0) + } + if !approxEqual(change.AvgProvisionedDisk, 120.0, 1e-9) { + t.Fatalf("unexpected vm-change AvgProvisionedDisk: got %.12f want %.12f", change.AvgProvisionedDisk, 120.0) + } + if !approxEqual(change.PoolTinPct, 100.0/3.0, 1e-9) || !approxEqual(change.PoolSilverPct, 100.0/3.0, 1e-9) || !approxEqual(change.PoolGoldPct, 100.0/3.0, 1e-9) { + t.Fatalf("unexpected vm-change pool percentages: tin=%.12f silver=%.12f gold=%.12f", change.PoolTinPct, change.PoolSilverPct, change.PoolGoldPct) + } +} + +type summaryTotalsByVcenter struct { + Vcenter string `db:"vcenter"` + VmCount int64 `db:"vm_count"` + VcpuTotal int64 `db:"vcpu_total"` + RamTotal int64 `db:"ram_total"` +} + +func newTasksTestCronTaskForAggregateFlow(t *testing.T, dbConn *sqlx.DB) *CronTask { + t.Helper() + task := newTasksTestCronTask(dbConn) + cfg := &settings.Settings{Values: &settings.SettingsYML{}} + asyncReports := false + cfg.Values.Settings.AsyncReportGeneration = &asyncReports + cfg.Values.Settings.ReportsDir = t.TempDir() + cfg.Values.Settings.MonthlyAggregationGranularity = "daily" + cfg.Values.Settings.ScheduledAggregationEngine = "go" + task.Settings = cfg + return task +} + +func assertSummaryCacheMatchesByVcenter(t *testing.T, ctx context.Context, dbConn *sqlx.DB, summaryTable, snapshotType string, snapshotTime int64) { + t.Helper() + + sql := fmt.Sprintf(` +SELECT + "Vcenter" AS vcenter, + COUNT(1) AS vm_count, + CAST(COALESCE(SUM(COALESCE("AvgVcpuCount","VcpuCount")),0) AS BIGINT) AS vcpu_total, + CAST(COALESCE(SUM(COALESCE("AvgRamGB","RamGB")),0) AS BIGINT) AS ram_total +FROM %s +GROUP BY "Vcenter" +`, summaryTable) + var expected []summaryTotalsByVcenter + if err := dbConn.SelectContext(ctx, &expected, sql); err != nil { + t.Fatalf("failed to load expected summary totals: %v", err) + } + if len(expected) == 0 { + t.Fatal("expected non-empty summary totals") + } + + cacheCountQuery := dbConn.Rebind(` +SELECT COUNT(1) +FROM vcenter_aggregate_totals +WHERE "SnapshotType" = ? AND "SnapshotTime" = ? +`) + var cacheCount int + if err := dbConn.GetContext(ctx, &cacheCount, cacheCountQuery, snapshotType, snapshotTime); err != nil { + t.Fatalf("failed to count cache rows: %v", err) + } + if cacheCount != len(expected) { + t.Fatalf("unexpected cache row count: got %d want %d", cacheCount, len(expected)) + } + + for _, exp := range expected { + rows, err := db.ListVcenterAggregateTotals(ctx, dbConn, exp.Vcenter, snapshotType, 10) + if err != nil { + t.Fatalf("ListVcenterAggregateTotals failed for %s/%s: %v", exp.Vcenter, snapshotType, err) + } + var got *db.VcenterTotalRow + for i := range rows { + if rows[i].SnapshotTime == snapshotTime { + got = &rows[i] + break + } + } + if got == nil { + t.Fatalf("missing cache row for vcenter=%s snapshot_type=%s snapshot_time=%d", exp.Vcenter, snapshotType, snapshotTime) + } + if got.VmCount != exp.VmCount || got.VcpuTotal != exp.VcpuTotal || got.RamTotalGB != exp.RamTotal { + t.Fatalf( + "cache mismatch for vcenter=%s snapshot_type=%s: got(vm=%d vcpu=%d ram=%d) want(vm=%d vcpu=%d ram=%d)", + exp.Vcenter, snapshotType, + got.VmCount, got.VcpuTotal, got.RamTotalGB, + exp.VmCount, exp.VcpuTotal, exp.RamTotal, + ) + } + } +} + +func assertRollupTotalSamplesForVcenter(t *testing.T, ctx context.Context, dbConn *sqlx.DB, dayUnix int64, vcenter string, wantTotalSamples int64) { + t.Helper() + + query := dbConn.Rebind(` +SELECT "TotalSamples" +FROM vm_daily_rollup +WHERE "Date" = ? AND "Vcenter" = ? +`) + var got []int64 + if err := dbConn.SelectContext(ctx, &got, query, dayUnix, vcenter); err != nil { + t.Fatalf("failed to read rollup total samples for %s: %v", vcenter, err) + } + if len(got) == 0 { + t.Fatalf("no rollup rows found for vcenter=%s date=%d", vcenter, dayUnix) + } + for _, value := range got { + if value != wantTotalSamples { + t.Fatalf("unexpected rollup TotalSamples for vcenter=%s: got %d want %d (rows=%v)", vcenter, value, wantTotalSamples, got) + } + } +} + +func assertSnapshotRegistryRow(t *testing.T, ctx context.Context, dbConn *sqlx.DB, snapshotType, tableName string, snapshotTime int64, snapshotCount int64) { + t.Helper() + + var row struct { + SnapshotType string `db:"snapshot_type"` + TableName string `db:"table_name"` + SnapshotTime int64 `db:"snapshot_time"` + SnapshotCount int64 `db:"snapshot_count"` + } + query := dbConn.Rebind(` +SELECT snapshot_type, table_name, snapshot_time, snapshot_count +FROM snapshot_registry +WHERE table_name = ? +`) + if err := dbConn.GetContext(ctx, &row, query, tableName); err != nil { + t.Fatalf("failed to load snapshot_registry row for table %s: %v", tableName, err) + } + if row.SnapshotType != snapshotType { + t.Fatalf("unexpected snapshot type for table %s: got %s want %s", tableName, row.SnapshotType, snapshotType) + } + if row.SnapshotTime != snapshotTime { + t.Fatalf("unexpected snapshot time for table %s: got %d want %d", tableName, row.SnapshotTime, snapshotTime) + } + if row.SnapshotCount != snapshotCount { + t.Fatalf("unexpected snapshot count for table %s: got %d want %d", tableName, row.SnapshotCount, snapshotCount) + } +} diff --git a/internal/tasks/aggregation_parity_test.go b/internal/tasks/aggregation_parity_test.go index 09b0684..4a44ec2 100644 --- a/internal/tasks/aggregation_parity_test.go +++ b/internal/tasks/aggregation_parity_test.go @@ -9,17 +9,19 @@ import ( "testing" "time" "vctp/db" + "vctp/db/queries" "github.com/jmoiron/sqlx" ) type tasksTestDatabase struct { - dbConn *sqlx.DB - logger *slog.Logger + dbConn *sqlx.DB + logger *slog.Logger + querier db.Querier } func (d *tasksTestDatabase) DB() *sqlx.DB { return d.dbConn } -func (d *tasksTestDatabase) Queries() db.Querier { return nil } +func (d *tasksTestDatabase) Queries() db.Querier { return d.querier } func (d *tasksTestDatabase) Logger() *slog.Logger { if d.logger != nil { return d.logger @@ -377,7 +379,7 @@ func newTasksTestCronTask(dbConn *sqlx.DB) *CronTask { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) return &CronTask{ Logger: logger, - Database: &tasksTestDatabase{dbConn: dbConn, logger: logger}, + Database: &tasksTestDatabase{dbConn: dbConn, logger: logger, querier: queries.New(dbConn.DB)}, } } diff --git a/internal/tasks/compatibility_integration_test.go b/internal/tasks/compatibility_integration_test.go new file mode 100644 index 0000000..92f62af --- /dev/null +++ b/internal/tasks/compatibility_integration_test.go @@ -0,0 +1,212 @@ +package tasks + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + "vctp/db" + "vctp/internal/settings" +) + +func TestSnapshotTableCompatModeSettingControlsTaskBehaviorFlag(t *testing.T) { + task := &CronTask{} + if !task.snapshotTableCompatModeEnabled() { + t.Fatal("expected default snapshot_table_compat_mode=true when settings are absent") + } + + task.Settings = &settings.Settings{Values: &settings.SettingsYML{}} + if !task.snapshotTableCompatModeEnabled() { + t.Fatal("expected default snapshot_table_compat_mode=true when value is unset") + } + + disabled := false + task.Settings.Values.Settings.SnapshotTableCompatMode = &disabled + if task.snapshotTableCompatModeEnabled() { + t.Fatal("expected snapshot_table_compat_mode=false to disable legacy snapshot-table writes") + } + + enabled := true + task.Settings.Values.Settings.SnapshotTableCompatMode = &enabled + if !task.snapshotTableCompatModeEnabled() { + t.Fatal("expected snapshot_table_compat_mode=true to enable legacy snapshot-table writes") + } +} + +func TestManualDailyAggregate_SQLFallback_LegacyTablesAndReport(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTaskForAggregateFlow(t, dbConn) + t.Setenv("DAILY_AGG_SQL", "1") + t.Setenv("DAILY_AGG_GO", "") + + dayStart := time.Date(2026, time.March, 15, 0, 0, 0, 0, time.UTC) + t1 := dayStart.Add(1 * time.Hour).Unix() + t2 := dayStart.Add(2 * time.Hour).Unix() + + table1, err := hourlyInventoryTableName(time.Unix(t1, 0).UTC()) + if err != nil { + t.Fatalf("failed to build first hourly table name: %v", err) + } + table2, err := hourlyInventoryTableName(time.Unix(t2, 0).UTC()) + if err != nil { + t.Fatalf("failed to build second hourly table name: %v", err) + } + for _, table := range []string{table1, table2} { + if err := db.EnsureSnapshotTable(ctx, dbConn, table); err != nil { + t.Fatalf("failed to ensure hourly snapshot table %s: %v", table, err) + } + } + + seeds := []hourlySeedRow{ + {SnapshotTime: t1, Name: "vm-a", Vcenter: "vc-a", VmID: "vm-a", VmUUID: "uuid-a", ResourcePool: "Tin", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 100, VcpuCount: 2, RamGB: 8, CreationTime: dayStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-a", Vcenter: "vc-a", VmID: "vm-a", VmUUID: "uuid-a", ResourcePool: "Gold", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 120, VcpuCount: 4, RamGB: 8, CreationTime: dayStart.Add(-24 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + {SnapshotTime: t2, Name: "vm-b", Vcenter: "vc-a", VmID: "vm-b", VmUUID: "uuid-b", ResourcePool: "Bronze", Datacenter: "dc-a", Cluster: "cluster-a", Folder: "/prod", ProvisionedDisk: 40, VcpuCount: 1, RamGB: 4, CreationTime: dayStart.Add(-48 * time.Hour).Unix(), IsTemplate: "FALSE", PoweredOn: "TRUE", SrmPlaceholder: "FALSE"}, + } + for _, row := range seeds { + table, tableErr := hourlyInventoryTableName(time.Unix(row.SnapshotTime, 0).UTC()) + if tableErr != nil { + t.Fatalf("failed to build hourly table for seed row: %v", tableErr) + } + if err := insertHourlySnapshotSeedRow(ctx, dbConn, table, row); err != nil { + t.Fatalf("failed to insert hourly snapshot seed row: %v", err) + } + } + + if err := task.aggregateDailySummaryWithMode(ctx, dayStart, true, false); err != nil { + t.Fatalf("aggregateDailySummaryWithMode (legacy SQL fallback) failed: %v", err) + } + + summaryTable, err := dailySummaryTableName(dayStart) + if err != nil { + t.Fatalf("failed to build daily summary table name: %v", err) + } + rows, err := loadDailySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load daily summary rows: %v", err) + } + if len(rows) != 2 { + t.Fatalf("unexpected daily summary row count: got %d want %d", len(rows), 2) + } + + assertSnapshotRegistryRow(t, ctx, dbConn, "daily", summaryTable, dayStart.Unix(), int64(len(rows))) + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "daily", dayStart.Unix()) + + reportPath := filepath.Join(task.Settings.Values.Settings.ReportsDir, summaryTable+".xlsx") + if _, err := os.Stat(reportPath); err != nil { + t.Fatalf("expected daily report file at %s: %v", reportPath, err) + } +} + +func TestManualMonthlyAggregate_SQLFallback_LegacyTablesAndReport(t *testing.T) { + ctx := context.Background() + dbConn := newTasksTestDB(t) + task := newTasksTestCronTaskForAggregateFlow(t, dbConn) + t.Setenv("MONTHLY_AGG_SQL", "1") + t.Setenv("MONTHLY_AGG_GO", "") + + monthStart := time.Date(2026, time.April, 1, 0, 0, 0, 0, time.UTC) + day1 := monthStart.AddDate(0, 0, 2) + day2 := monthStart.AddDate(0, 0, 3) + + day1Table, err := dailySummaryTableName(day1) + if err != nil { + t.Fatalf("failed to build day1 summary table name: %v", err) + } + day2Table, err := dailySummaryTableName(day2) + if err != nil { + t.Fatalf("failed to build day2 summary table name: %v", err) + } + for _, table := range []string{day1Table, day2Table} { + if err := db.EnsureSummaryTable(ctx, dbConn, table); err != nil { + t.Fatalf("failed to ensure daily summary table %s: %v", table, err) + } + } + + seeds := []dailySeedRow{ + { + SnapshotTime: day1.Unix(), + Name: "vm-a", + Vcenter: "vc-a", + VmID: "vm-a", + VmUUID: "uuid-a", + ResourcePool: "Bronze", + Datacenter: "dc-a", + Cluster: "cluster-a", + Folder: "/prod", + ProvisionedDisk: 100, + VcpuCount: 2, + RamGB: 8, + CreationTime: monthStart.Add(-72 * time.Hour).Unix(), + IsTemplate: "FALSE", + PoweredOn: "TRUE", + SrmPlaceholder: "FALSE", + SamplesPresent: 2, + AvgVcpuCount: 2, + AvgRamGB: 8, + AvgProvisionedDisk: 100, + AvgIsPresent: 1.0, + PoolBronzePct: 100, + Bronze: 100, + }, + { + SnapshotTime: day2.Unix(), + Name: "vm-a", + Vcenter: "vc-a", + VmID: "vm-a", + VmUUID: "uuid-a", + ResourcePool: "Tin", + Datacenter: "dc-a", + Cluster: "cluster-a", + Folder: "/prod", + ProvisionedDisk: 120, + VcpuCount: 4, + RamGB: 12, + CreationTime: monthStart.Add(-72 * time.Hour).Unix(), + IsTemplate: "FALSE", + PoweredOn: "TRUE", + SrmPlaceholder: "FALSE", + SamplesPresent: 2, + AvgVcpuCount: 4, + AvgRamGB: 12, + AvgProvisionedDisk: 120, + AvgIsPresent: 1.0, + PoolTinPct: 100, + Tin: 100, + }, + } + for _, seed := range seeds { + targetTable := day1Table + if seed.SnapshotTime == day2.Unix() { + targetTable = day2Table + } + if err := insertDailySummarySeedRow(ctx, dbConn, targetTable, seed); err != nil { + t.Fatalf("failed to insert daily summary seed row: %v", err) + } + } + + if err := task.aggregateMonthlySummaryWithMode(ctx, monthStart, true, false); err != nil { + t.Fatalf("aggregateMonthlySummaryWithMode (legacy SQL fallback) failed: %v", err) + } + + summaryTable, err := monthlySummaryTableName(monthStart) + if err != nil { + t.Fatalf("failed to build monthly summary table name: %v", err) + } + rows, err := loadMonthlySummaryRows(ctx, dbConn, summaryTable) + if err != nil { + t.Fatalf("failed to load monthly summary rows: %v", err) + } + if len(rows) != 1 { + t.Fatalf("unexpected monthly summary row count: got %d want %d", len(rows), 1) + } + + assertSnapshotRegistryRow(t, ctx, dbConn, "monthly", summaryTable, monthStart.Unix(), int64(len(rows))) + assertSummaryCacheMatchesByVcenter(t, ctx, dbConn, summaryTable, "monthly", monthStart.Unix()) + + reportPath := filepath.Join(task.Settings.Values.Settings.ReportsDir, summaryTable+".xlsx") + if _, err := os.Stat(reportPath); err != nil { + t.Fatalf("expected monthly report file at %s: %v", reportPath, err) + } +} diff --git a/internal/tasks/monthlyAggregate.go b/internal/tasks/monthlyAggregate.go index e022512..b246bc2 100644 --- a/internal/tasks/monthlyAggregate.go +++ b/internal/tasks/monthlyAggregate.go @@ -219,6 +219,11 @@ func (c *CronTask) aggregateMonthlySummaryWithMode(ctx context.Context, targetMo if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth, rowCount); err != nil { c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable) } + if refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, monthlyTable, "monthly", monthStart.Unix()); err != nil { + c.Logger.Warn("failed to refresh vcenter monthly aggregate totals cache", "error", err, "table", monthlyTable) + } else { + c.Logger.Debug("refreshed vcenter monthly aggregate totals cache", "table", monthlyTable, "rows", refreshed) + } db.AnalyzeTableIfPostgres(ctx, dbConn, monthlyTable) @@ -275,6 +280,11 @@ func (c *CronTask) aggregateMonthlySummarySQLCanonical(ctx context.Context, mont if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil { c.Logger.Warn("failed to register monthly snapshot (SQL canonical)", "error", err, "table", summaryTable) } + if refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "monthly", monthStart.Unix()); err != nil { + c.Logger.Warn("failed to refresh vcenter monthly aggregate totals cache (SQL canonical)", "error", err, "table", summaryTable) + } else { + c.Logger.Debug("refreshed vcenter monthly aggregate totals cache", "table", summaryTable, "rows", refreshed) + } if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil { c.Logger.Warn("failed to generate monthly report (SQL canonical)", "error", err, "table", summaryTable) return err @@ -389,6 +399,11 @@ func (c *CronTask) aggregateMonthlySummaryGoHourly(ctx context.Context, monthSta if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil { c.Logger.Warn("failed to register monthly snapshot (Go hourly)", "error", err, "table", summaryTable) } + if refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "monthly", monthStart.Unix()); err != nil { + c.Logger.Warn("failed to refresh vcenter monthly aggregate totals cache (Go hourly)", "error", err, "table", summaryTable) + } else { + c.Logger.Debug("refreshed vcenter monthly aggregate totals cache", "table", summaryTable, "rows", refreshed) + } if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil { c.Logger.Warn("failed to generate monthly report (Go hourly)", "error", err, "table", summaryTable) return err @@ -478,6 +493,11 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil { c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", summaryTable) } + if refreshed, err := db.ReplaceVcenterAggregateTotalsFromSummary(ctx, dbConn, summaryTable, "monthly", monthStart.Unix()); err != nil { + c.Logger.Warn("failed to refresh vcenter monthly aggregate totals cache", "error", err, "table", summaryTable) + } else { + c.Logger.Debug("refreshed vcenter monthly aggregate totals cache", "table", summaryTable, "rows", refreshed) + } if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil { c.Logger.Warn("failed to generate monthly report (Go)", "error", err, "table", summaryTable) return err diff --git a/plan.md b/plan.md index aa4765c..c2a23b9 100644 --- a/plan.md +++ b/plan.md @@ -318,9 +318,9 @@ The target architecture is: ### 5. Validation and Quality Gates - [ ] Add golden-result tests for daily output parity (old vs new path). - [ ] Add golden-result tests for monthly output parity (old vs new path). -- [ ] Add lifecycle edge-case coverage (partial presence, missing create times, deletion refinement, pool and resource changes). -- [ ] Add integration tests for canonical write/read paths and totals cache correctness. -- [ ] Add compatibility tests for legacy table generation, reports, and rebuild flows. +- [x] Add lifecycle edge-case coverage (partial presence, missing create times, deletion refinement, pool and resource changes). +- [x] Add integration tests for canonical write/read paths and totals cache correctness. +- [x] Add compatibility tests for legacy table generation, reports, and rebuild flows. - [ ] Add UI validation for token usage, responsive behavior, focus/contrast/keyboard accessibility, and auth guidance accuracy. - [ ] Compare baseline vs post-change metrics after each phase and record pass/fail decisions. diff --git a/server/handler/snapshot_repair_integration_test.go b/server/handler/snapshot_repair_integration_test.go new file mode 100644 index 0000000..c1c14ba --- /dev/null +++ b/server/handler/snapshot_repair_integration_test.go @@ -0,0 +1,181 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + "vctp/db" + "vctp/db/queries" + "vctp/server/models" + + "github.com/jmoiron/sqlx" + _ "modernc.org/sqlite" +) + +type snapshotRepairTestDatabase struct { + dbConn *sqlx.DB + logger *slog.Logger +} + +func (d *snapshotRepairTestDatabase) DB() *sqlx.DB { return d.dbConn } +func (d *snapshotRepairTestDatabase) Queries() db.Querier { return queries.New(d.dbConn.DB) } +func (d *snapshotRepairTestDatabase) Logger() *slog.Logger { + if d.logger != nil { + return d.logger + } + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} +func (d *snapshotRepairTestDatabase) Close() error { return d.dbConn.Close() } + +func newSnapshotRepairTestDB(t *testing.T) *sqlx.DB { + t.Helper() + dbConn, err := sqlx.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("failed to open sqlite test db: %v", err) + } + t.Cleanup(func() { + _ = dbConn.Close() + }) + return dbConn +} + +func TestSnapshotRepairSuite_RebuildsRegistryTotalsAndLifecycle(t *testing.T) { + ctx := context.Background() + dbConn := newSnapshotRepairTestDB(t) + logger := newTestLogger() + h := &Handler{ + Logger: logger, + Database: &snapshotRepairTestDatabase{dbConn: dbConn, logger: logger}, + } + + dayStart := time.Date(2026, time.March, 16, 0, 0, 0, 0, time.UTC) + hourlyTs := dayStart.Add(2 * time.Hour).Unix() + hourlyTable := fmt.Sprintf("inventory_hourly_%d", hourlyTs) + dailyTable := fmt.Sprintf("inventory_daily_summary_%s", dayStart.Format("20060102")) + monthlyTable := fmt.Sprintf("inventory_monthly_summary_%s", dayStart.Format("200601")) + + if err := db.EnsureSnapshotTable(ctx, dbConn, hourlyTable); err != nil { + t.Fatalf("failed to ensure hourly table: %v", err) + } + if err := db.EnsureSummaryTable(ctx, dbConn, dailyTable); err != nil { + t.Fatalf("failed to ensure daily summary table: %v", err) + } + if err := db.EnsureSummaryTable(ctx, dbConn, monthlyTable); err != nil { + t.Fatalf("failed to ensure monthly summary table: %v", err) + } + + if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(` +INSERT INTO %s ( + "Name","Vcenter","VmId","VmUuid","CreationTime","DeletionTime","ResourcePool","Datacenter","Cluster","Folder", + "ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder","SnapshotTime" +) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +`, hourlyTable), + "vm-a", "vc-a", "vm-a", "uuid-a", dayStart.Add(-24*time.Hour).Unix(), int64(0), "Tin", "dc-a", "cluster-a", "/prod", + 100.0, int64(2), int64(8), "FALSE", "TRUE", "FALSE", hourlyTs, + ); err != nil { + t.Fatalf("failed to seed hourly table: %v", err) + } + + if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(` +INSERT INTO %s ( + "Name","Vcenter","VmId","VmUuid","CreationTime","DeletionTime","ResourcePool","Datacenter","Cluster","Folder", + "ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder","SnapshotTime","SamplesPresent", + "AvgVcpuCount","AvgRamGB","AvgProvisionedDisk","AvgIsPresent","PoolTinPct","PoolBronzePct","PoolSilverPct","PoolGoldPct","Tin","Bronze","Silver","Gold" +) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +`, dailyTable), + "vm-a", "vc-a", "vm-a", "uuid-a", int64(0), int64(0), "Tin", "dc-a", "cluster-a", "/prod", + 100.0, int64(2), int64(8), "FALSE", "TRUE", "FALSE", int64(0), int64(1), + 2.0, 8.0, 100.0, 1.0, 100.0, 0.0, 0.0, 0.0, 100.0, 0.0, 0.0, 0.0, + ); err != nil { + t.Fatalf("failed to seed daily summary table: %v", err) + } + + if _, err := dbConn.ExecContext(ctx, fmt.Sprintf(` +INSERT INTO %s ( + "Name","Vcenter","VmId","VmUuid","CreationTime","DeletionTime","ResourcePool","Datacenter","Cluster","Folder", + "ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder","SnapshotTime","SamplesPresent", + "AvgVcpuCount","AvgRamGB","AvgProvisionedDisk","AvgIsPresent","PoolTinPct","PoolBronzePct","PoolSilverPct","PoolGoldPct","Tin","Bronze","Silver","Gold" +) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +`, monthlyTable), + "vm-a", "vc-a", "vm-a", "uuid-a", int64(0), int64(0), "Tin", "dc-a", "cluster-a", "/prod", + 100.0, int64(2), int64(8), "FALSE", "TRUE", "FALSE", dayStart.Unix(), int64(1), + 2.0, 8.0, 100.0, 1.0, 100.0, 0.0, 0.0, 0.0, 100.0, 0.0, 0.0, 0.0, + ); err != nil { + t.Fatalf("failed to seed monthly summary table: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/api/snapshots/repair/all", nil) + rr := httptest.NewRecorder() + h.SnapshotRepairSuite(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d body=%s", http.StatusOK, rr.Code, rr.Body.String()) + } + + var payload models.SnapshotRepairSuiteResponse + if err := json.Unmarshal(rr.Body.Bytes(), &payload); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + if payload.Status != "OK" { + t.Fatalf("unexpected repair suite status: %q", payload.Status) + } + + dailyRepaired, err := strconv.Atoi(payload.DailyRepaired) + if err != nil { + t.Fatalf("failed to parse daily_repaired: %v", err) + } + if dailyRepaired < 1 { + t.Fatalf("expected at least one daily table repaired, got %d", dailyRepaired) + } + monthlyRefined, err := strconv.Atoi(payload.MonthlyRefined) + if err != nil { + t.Fatalf("failed to parse monthly_refined: %v", err) + } + if monthlyRefined < 1 { + t.Fatalf("expected at least one monthly table refined, got %d", monthlyRefined) + } + monthlyFailed, err := strconv.Atoi(payload.MonthlyFailed) + if err != nil { + t.Fatalf("failed to parse monthly_failed: %v", err) + } + if monthlyFailed != 0 { + t.Fatalf("expected monthly_failed=0, got %d", monthlyFailed) + } + + assertSnapshotRegistryTypeCount(t, ctx, dbConn, "hourly", 1) + assertSnapshotRegistryTypeCount(t, ctx, dbConn, "daily", 1) + assertSnapshotRegistryTypeCount(t, ctx, dbConn, "monthly", 1) + + var totalsRows int + if err := dbConn.GetContext(ctx, &totalsRows, `SELECT COUNT(1) FROM vcenter_totals WHERE "Vcenter" = ?`, "vc-a"); err != nil { + t.Fatalf("failed to query vcenter_totals: %v", err) + } + if totalsRows < 1 { + t.Fatalf("expected vcenter_totals to be backfilled, got %d rows", totalsRows) + } + + var dailySnapshotTime int64 + if err := dbConn.GetContext(ctx, &dailySnapshotTime, fmt.Sprintf(`SELECT COALESCE("SnapshotTime",0) FROM %s WHERE "Vcenter" = ? AND "VmId" = ?`, dailyTable), "vc-a", "vm-a"); err != nil { + t.Fatalf("failed to query repaired daily snapshot time: %v", err) + } + if dailySnapshotTime == 0 { + t.Fatal("expected repaired daily summary SnapshotTime to be backfilled") + } +} + +func assertSnapshotRegistryTypeCount(t *testing.T, ctx context.Context, dbConn *sqlx.DB, snapshotType string, want int) { + t.Helper() + var got int + if err := dbConn.GetContext(ctx, &got, `SELECT COUNT(1) FROM snapshot_registry WHERE snapshot_type = ?`, snapshotType); err != nil { + t.Fatalf("failed to query snapshot_registry for type %s: %v", snapshotType, err) + } + if got != want { + t.Fatalf("unexpected snapshot_registry count for %s: got %d want %d", snapshotType, got, want) + } +}