From 6da2da3e82090651bfefd188e745628bd768fc98 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Mon, 16 Feb 2026 09:21:00 +1100 Subject: [PATCH] Add PostgreSQL checkpoint functionality and update related database operations --- db/helpers.go | 41 +++++++++ internal/report/create.go | 14 +++- internal/report/snapshots.go | 16 ++-- internal/report/snapshots_pivot_test.go | 62 ++++++++++++++ internal/tasks/dailyAggregate.go | 20 +++-- server/router/docs/docs.go | 107 ++++++++++++++++++++++-- server/router/docs/swagger.json | 107 ++++++++++++++++++++++-- server/router/docs/swagger.yaml | 79 +++++++++++++++-- 8 files changed, 412 insertions(+), 34 deletions(-) create mode 100644 internal/report/snapshots_pivot_test.go diff --git a/db/helpers.go b/db/helpers.go index 9d480f1..c52c3c4 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -633,6 +633,47 @@ func CheckpointSQLite(ctx context.Context, dbConn *sqlx.DB) error { return nil } +// CheckpointPostgres requests a checkpoint when using PostgreSQL. No-op for other drivers. +// If the connected role lacks permission, the request is skipped without returning an error. +func CheckpointPostgres(ctx context.Context, dbConn *sqlx.DB) error { + driver := strings.ToLower(dbConn.DriverName()) + if driver != "pgx" && driver != "postgres" { + return nil + } + if ctx == nil { + ctx = context.Background() + } + start := time.Now() + slog.Debug("postgres checkpoint start") + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + _, err := dbConn.ExecContext(cctx, `CHECKPOINT`) + if err != nil { + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "must be superuser") || strings.Contains(msg, "pg_checkpoint") || strings.Contains(msg, "permission denied") { + slog.Debug("postgres checkpoint skipped (insufficient privilege)", "error", err) + return nil + } + slog.Warn("postgres checkpoint failed", "error", err, "duration", time.Since(start)) + return err + } + slog.Debug("postgres checkpoint complete", "duration", time.Since(start)) + return nil +} + +// CheckpointDatabase performs the checkpoint operation appropriate for the active DB driver. +// It returns the action name for logging. +func CheckpointDatabase(ctx context.Context, dbConn *sqlx.DB) (string, error) { + switch strings.ToLower(dbConn.DriverName()) { + case "sqlite": + return "sqlite_wal_checkpoint", CheckpointSQLite(ctx, dbConn) + case "pgx", "postgres": + return "postgres_checkpoint", CheckpointPostgres(ctx, dbConn) + default: + return "none", nil + } +} + // EnsureVmHourlyStats creates the shared per-snapshot cache table used by Go aggregations. func EnsureVmHourlyStats(ctx context.Context, dbConn *sqlx.DB) error { ddl := ` diff --git a/internal/report/create.go b/internal/report/create.go index dfb26e6..e05b4e4 100644 --- a/internal/report/create.go +++ b/internal/report/create.go @@ -279,6 +279,8 @@ func SetColAutoWidth(xlsx *excelize.File, sheetName string) error { if err != nil { return err } + const minColWidth = 10 + const maxColWidth = 80 for idx, col := range cols { largestWidth := 0 for _, rowCell := range col { @@ -287,12 +289,22 @@ func SetColAutoWidth(xlsx *excelize.File, sheetName string) error { largestWidth = cellWidth } } + // Keep a sane minimum so sheets that rely on computed content + // (for example pivot output populated by Excel) don't collapse to width 0. + if largestWidth < minColWidth { + largestWidth = minColWidth + } + if largestWidth > maxColWidth { + largestWidth = maxColWidth + } //fmt.Printf("SetColAutoWidth calculated largest width for column index '%d' is '%d'\n", idx, largestWidth) name, err := excelize.ColumnNumberToName(idx + 1) if err != nil { return err } - xlsx.SetColWidth(sheetName, name, name, float64(largestWidth)) + if err := xlsx.SetColWidth(sheetName, name, name, float64(largestWidth)); err != nil { + return err + } } // No errors at this point return nil diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index f1153d2..8bbd51d 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -928,7 +928,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st logger.Warn("summary worksheet skipped due to invalid data range", "table", tableName, "error", err) return } - dataRange := fmt.Sprintf("%s!A1:%s", quoteSheetName(dataSheet), endCell) + // excelize AddPivotTable expects unquoted sheet references like: + // "Snapshot Report!A1:Z999". Quoted sheet names cause pivot creation to fail. + dataRange := fmt.Sprintf("%s!A1:%s", dataSheet, endCell) lowerToHeader := make(map[string]string, len(headers)) for _, header := range headers { lowerToHeader[strings.ToLower(strings.TrimSpace(header))] = header @@ -943,7 +945,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st Title: "Sum of Avg vCPUs", TitleCell: "A1", PivotName: "PivotAvgVcpu", - PivotRange: fmt.Sprintf("%s!A3:H1000", quoteSheetName(summarySheet)), + PivotRange: fmt.Sprintf("%s!A3:H22", summarySheet), RowFields: []string{"Datacenter", "ResourcePool"}, DataField: "AvgVcpuCount", DataName: "Sum of Avg vCPUs", @@ -953,7 +955,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st Title: "Sum of Avg RAM", TitleCell: "J1", PivotName: "PivotAvgRam", - PivotRange: fmt.Sprintf("%s!J3:P1000", quoteSheetName(summarySheet)), + PivotRange: fmt.Sprintf("%s!J3:P22", summarySheet), RowFields: []string{"Datacenter"}, DataField: "AvgRamGB", DataName: "Sum of Avg RAM", @@ -961,9 +963,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st }, { Title: "Sum of prorated VM count", - TitleCell: "A1003", + TitleCell: "A23", PivotName: "PivotProratedVmCount", - PivotRange: fmt.Sprintf("%s!A1005:H2002", quoteSheetName(summarySheet)), + PivotRange: fmt.Sprintf("%s!A25:H44", summarySheet), RowFields: []string{"Datacenter"}, DataField: "AvgIsPresent", DataName: "Sum of prorated VM count", @@ -971,9 +973,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st }, { Title: "Count of VM Name", - TitleCell: "J1003", + TitleCell: "J23", PivotName: "PivotVmNameCount", - PivotRange: fmt.Sprintf("%s!J1005:P2002", quoteSheetName(summarySheet)), + PivotRange: fmt.Sprintf("%s!J25:P44", summarySheet), RowFields: []string{"Datacenter"}, DataField: "Name", DataName: "Count of VM Name", diff --git a/internal/report/snapshots_pivot_test.go b/internal/report/snapshots_pivot_test.go new file mode 100644 index 0000000..d0641e8 --- /dev/null +++ b/internal/report/snapshots_pivot_test.go @@ -0,0 +1,62 @@ +package report + +import ( + "io" + "log/slog" + "strings" + "testing" + + "github.com/xuri/excelize/v2" +) + +func TestAddSummaryPivotSheetCreatesPivotTables(t *testing.T) { + xlsx := excelize.NewFile() + const dataSheet = "Snapshot Report" + if err := xlsx.SetSheetName("Sheet1", dataSheet); err != nil { + t.Fatalf("SetSheetName failed: %v", err) + } + + headers := []string{"Name", "Datacenter", "ResourcePool", "AvgVcpuCount", "AvgRamGB", "AvgIsPresent"} + if err := xlsx.SetSheetRow(dataSheet, "A1", &headers); err != nil { + t.Fatalf("SetSheetRow header failed: %v", err) + } + + row1 := []interface{}{"vm-1", "dc-1", "pool-1", 4.0, 16.0, 1.0} + if err := xlsx.SetSheetRow(dataSheet, "A2", &row1); err != nil { + t.Fatalf("SetSheetRow data failed: %v", err) + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215") + + pivots, err := xlsx.GetPivotTables("Summary") + if err != nil { + t.Fatalf("GetPivotTables failed: %v", err) + } + if len(pivots) != 4 { + t.Fatalf("expected 4 pivot tables, got %d", len(pivots)) + } + + expectedNames := map[string]bool{ + "PivotAvgVcpu": false, + "PivotAvgRam": false, + "PivotProratedVmCount": false, + "PivotVmNameCount": false, + } + for _, pivot := range pivots { + if _, ok := expectedNames[pivot.Name]; ok { + expectedNames[pivot.Name] = true + } + if strings.Contains(pivot.DataRange, "'") { + t.Fatalf("pivot %q has quoted DataRange %q; expected unquoted sheet reference", pivot.Name, pivot.DataRange) + } + if strings.Contains(pivot.PivotTableRange, "'") { + t.Fatalf("pivot %q has quoted PivotTableRange %q; expected unquoted sheet reference", pivot.Name, pivot.PivotTableRange) + } + } + for name, seen := range expectedNames { + if !seen { + t.Fatalf("missing expected pivot table %q", name) + } + } +} diff --git a/internal/tasks/dailyAggregate.go b/internal/tasks/dailyAggregate.go index 3353a03..47e1e6f 100644 --- a/internal/tasks/dailyAggregate.go +++ b/internal/tasks/dailyAggregate.go @@ -202,11 +202,13 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } c.Logger.Debug("Generated daily report", "table", summaryTable, "duration", time.Since(reportStart)) checkpointStart := time.Now() - c.Logger.Debug("Checkpointing sqlite after daily aggregation", "table", summaryTable) - if err := db.CheckpointSQLite(ctx, dbConn); err != nil { - c.Logger.Warn("failed to checkpoint sqlite after daily aggregation", "error", err) + driver := strings.ToLower(dbConn.DriverName()) + c.Logger.Debug("Running database checkpoint after daily aggregation", "table", summaryTable, "driver", driver) + action, err := db.CheckpointDatabase(ctx, dbConn) + if err != nil { + c.Logger.Warn("failed to run database checkpoint after daily aggregation", "driver", driver, "action", action, "error", err) } else { - c.Logger.Debug("Checkpointed sqlite after daily aggregation", "table", summaryTable, "duration", time.Since(checkpointStart)) + c.Logger.Debug("Completed database checkpoint after daily aggregation", "table", summaryTable, "driver", driver, "action", action, "duration", time.Since(checkpointStart)) } c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable) @@ -520,11 +522,13 @@ LIMIT 1 } c.Logger.Debug("Generated daily report", "table", summaryTable, "duration", time.Since(reportStart)) checkpointStart := time.Now() - c.Logger.Debug("Checkpointing sqlite after daily aggregation", "table", summaryTable) - if err := db.CheckpointSQLite(ctx, dbConn); err != nil { - c.Logger.Warn("failed to checkpoint sqlite after daily aggregation (Go path)", "error", err) + driver := strings.ToLower(dbConn.DriverName()) + c.Logger.Debug("Running database checkpoint after daily aggregation", "table", summaryTable, "driver", driver) + action, err := db.CheckpointDatabase(ctx, dbConn) + if err != nil { + c.Logger.Warn("failed to run database checkpoint after daily aggregation (Go path)", "driver", driver, "action", action, "error", err) } else { - c.Logger.Debug("Checkpointed sqlite after daily aggregation", "table", summaryTable, "duration", time.Since(checkpointStart)) + c.Logger.Debug("Completed database checkpoint after daily aggregation", "table", summaryTable, "driver", driver, "action", action, "duration", time.Since(checkpointStart)) } c.Logger.Debug("Finished daily inventory aggregation (Go path)", diff --git a/server/router/docs/docs.go b/server/router/docs/docs.go index 9a15f87..b2c87be 100644 --- a/server/router/docs/docs.go +++ b/server/router/docs/docs.go @@ -390,7 +390,7 @@ const docTemplate = `{ }, "/api/import/vm": { "post": { - "description": "Imports existing VM inventory data in bulk.", + "description": "Deprecated: Imports existing VM inventory data in bulk.", "consumes": [ "application/json" ], @@ -400,7 +400,8 @@ const docTemplate = `{ "tags": [ "inventory" ], - "summary": "Import VMs", + "summary": "Import VMs (deprecated)", + "deprecated": true, "parameters": [ { "description": "Bulk import payload", @@ -430,14 +431,15 @@ const docTemplate = `{ }, "/api/inventory/vm/delete": { "delete": { - "description": "Removes a VM inventory entry by VM ID and datacenter name.", + "description": "Deprecated: Removes a VM inventory entry by VM ID and datacenter name.", "produces": [ "application/json" ], "tags": [ "inventory" ], - "summary": "Cleanup VM inventory entry", + "summary": "Cleanup VM inventory entry (deprecated)", + "deprecated": true, "parameters": [ { "type": "string", @@ -472,14 +474,15 @@ const docTemplate = `{ }, "/api/inventory/vm/update": { "post": { - "description": "Queries vCenter and updates inventory records with missing details.", + "description": "Deprecated: Queries vCenter and updates inventory records with missing details.", "produces": [ "application/json" ], "tags": [ "inventory" ], - "summary": "Refresh VM details", + "summary": "Refresh VM details (deprecated)", + "deprecated": true, "responses": { "200": { "description": "Update completed", @@ -779,6 +782,52 @@ const docTemplate = `{ } } }, + "/api/vcenters/cache/rebuild": { + "post": { + "description": "Rebuilds cached folder/resource-pool/host(cluster+datacenter) references from vCenter and rewrites the database cache tables.", + "produces": [ + "application/json" + ], + "tags": [ + "vcenters" + ], + "summary": "Rebuild vCenter object cache", + "parameters": [ + { + "type": "string", + "description": "Optional single vCenter URL to rebuild; defaults to all configured vCenters", + "name": "vcenter", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Cache rebuild summary", + "schema": { + "$ref": "#/definitions/models.VcenterCacheRebuildResponse" + } + }, + "400": { + "description": "Invalid request", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "405": { + "description": "Method not allowed", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "500": { + "description": "All rebuild attempts failed", + "schema": { + "$ref": "#/definitions/models.VcenterCacheRebuildResponse" + } + } + } + } + }, "/metrics": { "get": { "description": "Exposes Prometheus metrics for vctp.", @@ -1517,6 +1566,52 @@ const docTemplate = `{ "type": "string" } } + }, + "models.VcenterCacheRebuildResponse": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/models.VcenterCacheRebuildResult" + } + }, + "status": { + "type": "string" + }, + "succeeded": { + "type": "integer" + }, + "total": { + "type": "integer" + } + } + }, + "models.VcenterCacheRebuildResult": { + "type": "object", + "properties": { + "duration_seconds": { + "type": "number" + }, + "error": { + "type": "string" + }, + "folder_entries": { + "type": "integer" + }, + "host_entries": { + "type": "integer" + }, + "resource_pool_entries": { + "type": "integer" + }, + "vcenter": { + "type": "string" + } + } } } }` diff --git a/server/router/docs/swagger.json b/server/router/docs/swagger.json index 62a0eb3..9929b73 100644 --- a/server/router/docs/swagger.json +++ b/server/router/docs/swagger.json @@ -379,7 +379,7 @@ }, "/api/import/vm": { "post": { - "description": "Imports existing VM inventory data in bulk.", + "description": "Deprecated: Imports existing VM inventory data in bulk.", "consumes": [ "application/json" ], @@ -389,7 +389,8 @@ "tags": [ "inventory" ], - "summary": "Import VMs", + "summary": "Import VMs (deprecated)", + "deprecated": true, "parameters": [ { "description": "Bulk import payload", @@ -419,14 +420,15 @@ }, "/api/inventory/vm/delete": { "delete": { - "description": "Removes a VM inventory entry by VM ID and datacenter name.", + "description": "Deprecated: Removes a VM inventory entry by VM ID and datacenter name.", "produces": [ "application/json" ], "tags": [ "inventory" ], - "summary": "Cleanup VM inventory entry", + "summary": "Cleanup VM inventory entry (deprecated)", + "deprecated": true, "parameters": [ { "type": "string", @@ -461,14 +463,15 @@ }, "/api/inventory/vm/update": { "post": { - "description": "Queries vCenter and updates inventory records with missing details.", + "description": "Deprecated: Queries vCenter and updates inventory records with missing details.", "produces": [ "application/json" ], "tags": [ "inventory" ], - "summary": "Refresh VM details", + "summary": "Refresh VM details (deprecated)", + "deprecated": true, "responses": { "200": { "description": "Update completed", @@ -768,6 +771,52 @@ } } }, + "/api/vcenters/cache/rebuild": { + "post": { + "description": "Rebuilds cached folder/resource-pool/host(cluster+datacenter) references from vCenter and rewrites the database cache tables.", + "produces": [ + "application/json" + ], + "tags": [ + "vcenters" + ], + "summary": "Rebuild vCenter object cache", + "parameters": [ + { + "type": "string", + "description": "Optional single vCenter URL to rebuild; defaults to all configured vCenters", + "name": "vcenter", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Cache rebuild summary", + "schema": { + "$ref": "#/definitions/models.VcenterCacheRebuildResponse" + } + }, + "400": { + "description": "Invalid request", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "405": { + "description": "Method not allowed", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "500": { + "description": "All rebuild attempts failed", + "schema": { + "$ref": "#/definitions/models.VcenterCacheRebuildResponse" + } + } + } + } + }, "/metrics": { "get": { "description": "Exposes Prometheus metrics for vctp.", @@ -1506,6 +1555,52 @@ "type": "string" } } + }, + "models.VcenterCacheRebuildResponse": { + "type": "object", + "properties": { + "failed": { + "type": "integer" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/definitions/models.VcenterCacheRebuildResult" + } + }, + "status": { + "type": "string" + }, + "succeeded": { + "type": "integer" + }, + "total": { + "type": "integer" + } + } + }, + "models.VcenterCacheRebuildResult": { + "type": "object", + "properties": { + "duration_seconds": { + "type": "number" + }, + "error": { + "type": "string" + }, + "folder_entries": { + "type": "integer" + }, + "host_entries": { + "type": "integer" + }, + "resource_pool_entries": { + "type": "integer" + }, + "vcenter": { + "type": "string" + } + } } } } \ No newline at end of file diff --git a/server/router/docs/swagger.yaml b/server/router/docs/swagger.yaml index be95c1b..139aeef 100644 --- a/server/router/docs/swagger.yaml +++ b/server/router/docs/swagger.yaml @@ -288,6 +288,36 @@ definitions: status: type: string type: object + models.VcenterCacheRebuildResponse: + properties: + failed: + type: integer + results: + items: + $ref: '#/definitions/models.VcenterCacheRebuildResult' + type: array + status: + type: string + succeeded: + type: integer + total: + type: integer + type: object + models.VcenterCacheRebuildResult: + properties: + duration_seconds: + type: number + error: + type: string + folder_entries: + type: integer + host_entries: + type: integer + resource_pool_entries: + type: integer + vcenter: + type: string + type: object info: contact: {} paths: @@ -548,7 +578,8 @@ paths: post: consumes: - application/json - description: Imports existing VM inventory data in bulk. + deprecated: true + description: 'Deprecated: Imports existing VM inventory data in bulk.' parameters: - description: Bulk import payload in: body @@ -567,12 +598,14 @@ paths: description: Server error schema: $ref: '#/definitions/models.ErrorResponse' - summary: Import VMs + summary: Import VMs (deprecated) tags: - inventory /api/inventory/vm/delete: delete: - description: Removes a VM inventory entry by VM ID and datacenter name. + deprecated: true + description: 'Deprecated: Removes a VM inventory entry by VM ID and datacenter + name.' parameters: - description: VM ID in: query @@ -595,12 +628,14 @@ paths: description: Invalid request schema: $ref: '#/definitions/models.ErrorResponse' - summary: Cleanup VM inventory entry + summary: Cleanup VM inventory entry (deprecated) tags: - inventory /api/inventory/vm/update: post: - description: Queries vCenter and updates inventory records with missing details. + deprecated: true + description: 'Deprecated: Queries vCenter and updates inventory records with + missing details.' produces: - application/json responses: @@ -612,7 +647,7 @@ paths: description: Server error schema: $ref: '#/definitions/models.ErrorResponse' - summary: Refresh VM details + summary: Refresh VM details (deprecated) tags: - inventory /api/report/inventory: @@ -807,6 +842,38 @@ paths: summary: Run full snapshot repair suite tags: - snapshots + /api/vcenters/cache/rebuild: + post: + description: Rebuilds cached folder/resource-pool/host(cluster+datacenter) references + from vCenter and rewrites the database cache tables. + parameters: + - description: Optional single vCenter URL to rebuild; defaults to all configured + vCenters + in: query + name: vcenter + type: string + produces: + - application/json + responses: + "200": + description: Cache rebuild summary + schema: + $ref: '#/definitions/models.VcenterCacheRebuildResponse' + "400": + description: Invalid request + schema: + $ref: '#/definitions/models.ErrorResponse' + "405": + description: Method not allowed + schema: + $ref: '#/definitions/models.ErrorResponse' + "500": + description: All rebuild attempts failed + schema: + $ref: '#/definitions/models.VcenterCacheRebuildResponse' + summary: Rebuild vCenter object cache + tags: + - vcenters /metrics: get: description: Exposes Prometheus metrics for vctp.