Add PostgreSQL checkpoint functionality and update related database operations

db/helpers.go

@@ -633,6 +633,47 @@ func CheckpointSQLite(ctx context.Context, dbConn *sqlx.DB) error {
 	return nil
 }
 
+// CheckpointPostgres requests a checkpoint when using PostgreSQL. No-op for other drivers.
+// If the connected role lacks permission, the request is skipped without returning an error.
+func CheckpointPostgres(ctx context.Context, dbConn *sqlx.DB) error {
+	driver := strings.ToLower(dbConn.DriverName())
+	if driver != "pgx" && driver != "postgres" {
+		return nil
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	start := time.Now()
+	slog.Debug("postgres checkpoint start")
+	cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	_, err := dbConn.ExecContext(cctx, `CHECKPOINT`)
+	if err != nil {
+		msg := strings.ToLower(err.Error())
+		if strings.Contains(msg, "must be superuser") || strings.Contains(msg, "pg_checkpoint") || strings.Contains(msg, "permission denied") {
+			slog.Debug("postgres checkpoint skipped (insufficient privilege)", "error", err)
+			return nil
+		}
+		slog.Warn("postgres checkpoint failed", "error", err, "duration", time.Since(start))
+		return err
+	}
+	slog.Debug("postgres checkpoint complete", "duration", time.Since(start))
+	return nil
+}
+
+// CheckpointDatabase performs the checkpoint operation appropriate for the active DB driver.
+// It returns the action name for logging.
+func CheckpointDatabase(ctx context.Context, dbConn *sqlx.DB) (string, error) {
+	switch strings.ToLower(dbConn.DriverName()) {
+	case "sqlite":
+		return "sqlite_wal_checkpoint", CheckpointSQLite(ctx, dbConn)
+	case "pgx", "postgres":
+		return "postgres_checkpoint", CheckpointPostgres(ctx, dbConn)
+	default:
+		return "none", nil
+	}
+}
+
 // EnsureVmHourlyStats creates the shared per-snapshot cache table used by Go aggregations.
 func EnsureVmHourlyStats(ctx context.Context, dbConn *sqlx.DB) error {
 	ddl := `

internal/report/create.go

@@ -279,6 +279,8 @@ func SetColAutoWidth(xlsx *excelize.File, sheetName string) error {
 	if err != nil {
 		return err
 	}
+	const minColWidth = 10
+	const maxColWidth = 80
 	for idx, col := range cols {
 		largestWidth := 0
 		for _, rowCell := range col {
@@ -287,12 +289,22 @@ func SetColAutoWidth(xlsx *excelize.File, sheetName string) error {
 				largestWidth = cellWidth
 			}
 		}
+		// Keep a sane minimum so sheets that rely on computed content
+		// (for example pivot output populated by Excel) don't collapse to width 0.
+		if largestWidth < minColWidth {
+			largestWidth = minColWidth
+		}
+		if largestWidth > maxColWidth {
+			largestWidth = maxColWidth
+		}
 		//fmt.Printf("SetColAutoWidth calculated largest width for column index '%d' is '%d'\n", idx, largestWidth)
 		name, err := excelize.ColumnNumberToName(idx + 1)
 		if err != nil {
 			return err
 		}
-		xlsx.SetColWidth(sheetName, name, name, float64(largestWidth))
+		if err := xlsx.SetColWidth(sheetName, name, name, float64(largestWidth)); err != nil {
+			return err
+		}
 	}
 	// No errors at this point
 	return nil

internal/report/snapshots.go

@@ -928,7 +928,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
 		logger.Warn("summary worksheet skipped due to invalid data range", "table", tableName, "error", err)
 		return
 	}
-	dataRange := fmt.Sprintf("%s!A1:%s", quoteSheetName(dataSheet), endCell)
+	// excelize AddPivotTable expects unquoted sheet references like:
+	// "Snapshot Report!A1:Z999". Quoted sheet names cause pivot creation to fail.
+	dataRange := fmt.Sprintf("%s!A1:%s", dataSheet, endCell)
 	lowerToHeader := make(map[string]string, len(headers))
 	for _, header := range headers {
 		lowerToHeader[strings.ToLower(strings.TrimSpace(header))] = header
@@ -943,7 +945,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
 			Title:       "Sum of Avg vCPUs",
 			TitleCell:   "A1",
 			PivotName:   "PivotAvgVcpu",
-			PivotRange:  fmt.Sprintf("%s!A3:H1000", quoteSheetName(summarySheet)),
+			PivotRange:  fmt.Sprintf("%s!A3:H22", summarySheet),
 			RowFields:   []string{"Datacenter", "ResourcePool"},
 			DataField:   "AvgVcpuCount",
 			DataName:    "Sum of Avg vCPUs",
@@ -953,7 +955,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
 			Title:       "Sum of Avg RAM",
 			TitleCell:   "J1",
 			PivotName:   "PivotAvgRam",
-			PivotRange:  fmt.Sprintf("%s!J3:P1000", quoteSheetName(summarySheet)),
+			PivotRange:  fmt.Sprintf("%s!J3:P22", summarySheet),
 			RowFields:   []string{"Datacenter"},
 			DataField:   "AvgRamGB",
 			DataName:    "Sum of Avg RAM",
@@ -961,9 +963,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
 		},
 		{
 			Title:       "Sum of prorated VM count",
-			TitleCell:   "A1003",
+			TitleCell:   "A23",
 			PivotName:   "PivotProratedVmCount",
-			PivotRange:  fmt.Sprintf("%s!A1005:H2002", quoteSheetName(summarySheet)),
+			PivotRange:  fmt.Sprintf("%s!A25:H44", summarySheet),
 			RowFields:   []string{"Datacenter"},
 			DataField:   "AvgIsPresent",
 			DataName:    "Sum of prorated VM count",
@@ -971,9 +973,9 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
 		},
 		{
 			Title:       "Count of VM Name",
-			TitleCell:   "J1003",
+			TitleCell:   "J23",
 			PivotName:   "PivotVmNameCount",
-			PivotRange:  fmt.Sprintf("%s!J1005:P2002", quoteSheetName(summarySheet)),
+			PivotRange:  fmt.Sprintf("%s!J25:P44", summarySheet),
 			RowFields:   []string{"Datacenter"},
 			DataField:   "Name",
 			DataName:    "Count of VM Name",

internal/report/snapshots_pivot_test.go

@@ -0,0 +1,62 @@
+package report
+
+import (
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/xuri/excelize/v2"
+)
+
+func TestAddSummaryPivotSheetCreatesPivotTables(t *testing.T) {
+	xlsx := excelize.NewFile()
+	const dataSheet = "Snapshot Report"
+	if err := xlsx.SetSheetName("Sheet1", dataSheet); err != nil {
+		t.Fatalf("SetSheetName failed: %v", err)
+	}
+
+	headers := []string{"Name", "Datacenter", "ResourcePool", "AvgVcpuCount", "AvgRamGB", "AvgIsPresent"}
+	if err := xlsx.SetSheetRow(dataSheet, "A1", &headers); err != nil {
+		t.Fatalf("SetSheetRow header failed: %v", err)
+	}
+
+	row1 := []interface{}{"vm-1", "dc-1", "pool-1", 4.0, 16.0, 1.0}
+	if err := xlsx.SetSheetRow(dataSheet, "A2", &row1); err != nil {
+		t.Fatalf("SetSheetRow data failed: %v", err)
+	}
+
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215")
+
+	pivots, err := xlsx.GetPivotTables("Summary")
+	if err != nil {
+		t.Fatalf("GetPivotTables failed: %v", err)
+	}
+	if len(pivots) != 4 {
+		t.Fatalf("expected 4 pivot tables, got %d", len(pivots))
+	}
+
+	expectedNames := map[string]bool{
+		"PivotAvgVcpu":         false,
+		"PivotAvgRam":          false,
+		"PivotProratedVmCount": false,
+		"PivotVmNameCount":     false,
+	}
+	for _, pivot := range pivots {
+		if _, ok := expectedNames[pivot.Name]; ok {
+			expectedNames[pivot.Name] = true
+		}
+		if strings.Contains(pivot.DataRange, "'") {
+			t.Fatalf("pivot %q has quoted DataRange %q; expected unquoted sheet reference", pivot.Name, pivot.DataRange)
+		}
+		if strings.Contains(pivot.PivotTableRange, "'") {
+			t.Fatalf("pivot %q has quoted PivotTableRange %q; expected unquoted sheet reference", pivot.Name, pivot.PivotTableRange)
+		}
+	}
+	for name, seen := range expectedNames {
+		if !seen {
+			t.Fatalf("missing expected pivot table %q", name)
+		}
+	}
+}

internal/tasks/dailyAggregate.go

@@ -202,11 +202,13 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
 	}
 	c.Logger.Debug("Generated daily report", "table", summaryTable, "duration", time.Since(reportStart))
 	checkpointStart := time.Now()
-	c.Logger.Debug("Checkpointing sqlite after daily aggregation", "table", summaryTable)
+	driver := strings.ToLower(dbConn.DriverName())
-	if err := db.CheckpointSQLite(ctx, dbConn); err != nil {
+	c.Logger.Debug("Running database checkpoint after daily aggregation", "table", summaryTable, "driver", driver)
-		c.Logger.Warn("failed to checkpoint sqlite after daily aggregation", "error", err)
+	action, err := db.CheckpointDatabase(ctx, dbConn)
+	if err != nil {
+		c.Logger.Warn("failed to run database checkpoint after daily aggregation", "driver", driver, "action", action, "error", err)
 	} else {
-		c.Logger.Debug("Checkpointed sqlite after daily aggregation", "table", summaryTable, "duration", time.Since(checkpointStart))
+		c.Logger.Debug("Completed database checkpoint after daily aggregation", "table", summaryTable, "driver", driver, "action", action, "duration", time.Since(checkpointStart))
 	}
 
 	c.Logger.Debug("Finished daily inventory aggregation", "summary_table", summaryTable)
@@ -520,11 +522,13 @@ LIMIT 1
 	}
 	c.Logger.Debug("Generated daily report", "table", summaryTable, "duration", time.Since(reportStart))
 	checkpointStart := time.Now()
-	c.Logger.Debug("Checkpointing sqlite after daily aggregation", "table", summaryTable)
+	driver := strings.ToLower(dbConn.DriverName())
-	if err := db.CheckpointSQLite(ctx, dbConn); err != nil {
+	c.Logger.Debug("Running database checkpoint after daily aggregation", "table", summaryTable, "driver", driver)
-		c.Logger.Warn("failed to checkpoint sqlite after daily aggregation (Go path)", "error", err)
+	action, err := db.CheckpointDatabase(ctx, dbConn)
+	if err != nil {
+		c.Logger.Warn("failed to run database checkpoint after daily aggregation (Go path)", "driver", driver, "action", action, "error", err)
 	} else {
-		c.Logger.Debug("Checkpointed sqlite after daily aggregation", "table", summaryTable, "duration", time.Since(checkpointStart))
+		c.Logger.Debug("Completed database checkpoint after daily aggregation", "table", summaryTable, "driver", driver, "action", action, "duration", time.Since(checkpointStart))
 	}
 
 	c.Logger.Debug("Finished daily inventory aggregation (Go path)",

server/router/docs/docs.go

@@ -390,7 +390,7 @@ const docTemplate = `{
         },
         "/api/import/vm": {
             "post": {
-                "description": "Imports existing VM inventory data in bulk.",
+                "description": "Deprecated: Imports existing VM inventory data in bulk.",
                 "consumes": [
                     "application/json"
                 ],
@@ -400,7 +400,8 @@ const docTemplate = `{
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Import VMs",
+                "summary": "Import VMs (deprecated)",
+                "deprecated": true,
                 "parameters": [
                     {
                         "description": "Bulk import payload",
@@ -430,14 +431,15 @@ const docTemplate = `{
         },
         "/api/inventory/vm/delete": {
             "delete": {
-                "description": "Removes a VM inventory entry by VM ID and datacenter name.",
+                "description": "Deprecated: Removes a VM inventory entry by VM ID and datacenter name.",
                 "produces": [
                     "application/json"
                 ],
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Cleanup VM inventory entry",
+                "summary": "Cleanup VM inventory entry (deprecated)",
+                "deprecated": true,
                 "parameters": [
                     {
                         "type": "string",
@@ -472,14 +474,15 @@ const docTemplate = `{
         },
         "/api/inventory/vm/update": {
             "post": {
-                "description": "Queries vCenter and updates inventory records with missing details.",
+                "description": "Deprecated: Queries vCenter and updates inventory records with missing details.",
                 "produces": [
                     "application/json"
                 ],
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Refresh VM details",
+                "summary": "Refresh VM details (deprecated)",
+                "deprecated": true,
                 "responses": {
                     "200": {
                         "description": "Update completed",
@@ -779,6 +782,52 @@ const docTemplate = `{
                 }
             }
         },
+        "/api/vcenters/cache/rebuild": {
+            "post": {
+                "description": "Rebuilds cached folder/resource-pool/host(cluster+datacenter) references from vCenter and rewrites the database cache tables.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "vcenters"
+                ],
+                "summary": "Rebuild vCenter object cache",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Optional single vCenter URL to rebuild; defaults to all configured vCenters",
+                        "name": "vcenter",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Cache rebuild summary",
+                        "schema": {
+                            "$ref": "#/definitions/models.VcenterCacheRebuildResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Invalid request",
+                        "schema": {
+                            "$ref": "#/definitions/models.ErrorResponse"
+                        }
+                    },
+                    "405": {
+                        "description": "Method not allowed",
+                        "schema": {
+                            "$ref": "#/definitions/models.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "All rebuild attempts failed",
+                        "schema": {
+                            "$ref": "#/definitions/models.VcenterCacheRebuildResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/metrics": {
             "get": {
                 "description": "Exposes Prometheus metrics for vctp.",
@@ -1517,6 +1566,52 @@ const docTemplate = `{
                     "type": "string"
                 }
             }
+        },
+        "models.VcenterCacheRebuildResponse": {
+            "type": "object",
+            "properties": {
+                "failed": {
+                    "type": "integer"
+                },
+                "results": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/models.VcenterCacheRebuildResult"
+                    }
+                },
+                "status": {
+                    "type": "string"
+                },
+                "succeeded": {
+                    "type": "integer"
+                },
+                "total": {
+                    "type": "integer"
+                }
+            }
+        },
+        "models.VcenterCacheRebuildResult": {
+            "type": "object",
+            "properties": {
+                "duration_seconds": {
+                    "type": "number"
+                },
+                "error": {
+                    "type": "string"
+                },
+                "folder_entries": {
+                    "type": "integer"
+                },
+                "host_entries": {
+                    "type": "integer"
+                },
+                "resource_pool_entries": {
+                    "type": "integer"
+                },
+                "vcenter": {
+                    "type": "string"
+                }
+            }
         }
     }
 }`

server/router/docs/swagger.json

@@ -379,7 +379,7 @@
         },
         "/api/import/vm": {
             "post": {
-                "description": "Imports existing VM inventory data in bulk.",
+                "description": "Deprecated: Imports existing VM inventory data in bulk.",
                 "consumes": [
                     "application/json"
                 ],
@@ -389,7 +389,8 @@
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Import VMs",
+                "summary": "Import VMs (deprecated)",
+                "deprecated": true,
                 "parameters": [
                     {
                         "description": "Bulk import payload",
@@ -419,14 +420,15 @@
         },
         "/api/inventory/vm/delete": {
             "delete": {
-                "description": "Removes a VM inventory entry by VM ID and datacenter name.",
+                "description": "Deprecated: Removes a VM inventory entry by VM ID and datacenter name.",
                 "produces": [
                     "application/json"
                 ],
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Cleanup VM inventory entry",
+                "summary": "Cleanup VM inventory entry (deprecated)",
+                "deprecated": true,
                 "parameters": [
                     {
                         "type": "string",
@@ -461,14 +463,15 @@
         },
         "/api/inventory/vm/update": {
             "post": {
-                "description": "Queries vCenter and updates inventory records with missing details.",
+                "description": "Deprecated: Queries vCenter and updates inventory records with missing details.",
                 "produces": [
                     "application/json"
                 ],
                 "tags": [
                     "inventory"
                 ],
-                "summary": "Refresh VM details",
+                "summary": "Refresh VM details (deprecated)",
+                "deprecated": true,
                 "responses": {
                     "200": {
                         "description": "Update completed",
@@ -768,6 +771,52 @@
                 }
             }
         },
+        "/api/vcenters/cache/rebuild": {
+            "post": {
+                "description": "Rebuilds cached folder/resource-pool/host(cluster+datacenter) references from vCenter and rewrites the database cache tables.",
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "vcenters"
+                ],
+                "summary": "Rebuild vCenter object cache",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Optional single vCenter URL to rebuild; defaults to all configured vCenters",
+                        "name": "vcenter",
+                        "in": "query"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "Cache rebuild summary",
+                        "schema": {
+                            "$ref": "#/definitions/models.VcenterCacheRebuildResponse"
+                        }
+                    },
+                    "400": {
+                        "description": "Invalid request",
+                        "schema": {
+                            "$ref": "#/definitions/models.ErrorResponse"
+                        }
+                    },
+                    "405": {
+                        "description": "Method not allowed",
+                        "schema": {
+                            "$ref": "#/definitions/models.ErrorResponse"
+                        }
+                    },
+                    "500": {
+                        "description": "All rebuild attempts failed",
+                        "schema": {
+                            "$ref": "#/definitions/models.VcenterCacheRebuildResponse"
+                        }
+                    }
+                }
+            }
+        },
         "/metrics": {
             "get": {
                 "description": "Exposes Prometheus metrics for vctp.",
@@ -1506,6 +1555,52 @@
                     "type": "string"
                 }
             }
+        },
+        "models.VcenterCacheRebuildResponse": {
+            "type": "object",
+            "properties": {
+                "failed": {
+                    "type": "integer"
+                },
+                "results": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/models.VcenterCacheRebuildResult"
+                    }
+                },
+                "status": {
+                    "type": "string"
+                },
+                "succeeded": {
+                    "type": "integer"
+                },
+                "total": {
+                    "type": "integer"
+                }
+            }
+        },
+        "models.VcenterCacheRebuildResult": {
+            "type": "object",
+            "properties": {
+                "duration_seconds": {
+                    "type": "number"
+                },
+                "error": {
+                    "type": "string"
+                },
+                "folder_entries": {
+                    "type": "integer"
+                },
+                "host_entries": {
+                    "type": "integer"
+                },
+                "resource_pool_entries": {
+                    "type": "integer"
+                },
+                "vcenter": {
+                    "type": "string"
+                }
+            }
         }
     }
 }

server/router/docs/swagger.yaml

@@ -288,6 +288,36 @@ definitions:
       status:
         type: string
     type: object
+  models.VcenterCacheRebuildResponse:
+    properties:
+      failed:
+        type: integer
+      results:
+        items:
+          $ref: '#/definitions/models.VcenterCacheRebuildResult'
+        type: array
+      status:
+        type: string
+      succeeded:
+        type: integer
+      total:
+        type: integer
+    type: object
+  models.VcenterCacheRebuildResult:
+    properties:
+      duration_seconds:
+        type: number
+      error:
+        type: string
+      folder_entries:
+        type: integer
+      host_entries:
+        type: integer
+      resource_pool_entries:
+        type: integer
+      vcenter:
+        type: string
+    type: object
 info:
   contact: {}
 paths:
@@ -548,7 +578,8 @@ paths:
     post:
       consumes:
       - application/json
-      description: Imports existing VM inventory data in bulk.
+      deprecated: true
+      description: 'Deprecated: Imports existing VM inventory data in bulk.'
       parameters:
       - description: Bulk import payload
         in: body
@@ -567,12 +598,14 @@ paths:
           description: Server error
           schema:
             $ref: '#/definitions/models.ErrorResponse'
-      summary: Import VMs
+      summary: Import VMs (deprecated)
       tags:
       - inventory
   /api/inventory/vm/delete:
     delete:
-      description: Removes a VM inventory entry by VM ID and datacenter name.
+      deprecated: true
+      description: 'Deprecated: Removes a VM inventory entry by VM ID and datacenter
+        name.'
       parameters:
       - description: VM ID
         in: query
@@ -595,12 +628,14 @@ paths:
           description: Invalid request
           schema:
             $ref: '#/definitions/models.ErrorResponse'
-      summary: Cleanup VM inventory entry
+      summary: Cleanup VM inventory entry (deprecated)
       tags:
       - inventory
   /api/inventory/vm/update:
     post:
-      description: Queries vCenter and updates inventory records with missing details.
+      deprecated: true
+      description: 'Deprecated: Queries vCenter and updates inventory records with
+        missing details.'
       produces:
       - application/json
       responses:
@@ -612,7 +647,7 @@ paths:
           description: Server error
           schema:
             $ref: '#/definitions/models.ErrorResponse'
-      summary: Refresh VM details
+      summary: Refresh VM details (deprecated)
       tags:
       - inventory
   /api/report/inventory:
@@ -807,6 +842,38 @@ paths:
       summary: Run full snapshot repair suite
       tags:
       - snapshots
+  /api/vcenters/cache/rebuild:
+    post:
+      description: Rebuilds cached folder/resource-pool/host(cluster+datacenter) references
+        from vCenter and rewrites the database cache tables.
+      parameters:
+      - description: Optional single vCenter URL to rebuild; defaults to all configured
+          vCenters
+        in: query
+        name: vcenter
+        type: string
+      produces:
+      - application/json
+      responses:
+        "200":
+          description: Cache rebuild summary
+          schema:
+            $ref: '#/definitions/models.VcenterCacheRebuildResponse'
+        "400":
+          description: Invalid request
+          schema:
+            $ref: '#/definitions/models.ErrorResponse'
+        "405":
+          description: Method not allowed
+          schema:
+            $ref: '#/definitions/models.ErrorResponse'
+        "500":
+          description: All rebuild attempts failed
+          schema:
+            $ref: '#/definitions/models.VcenterCacheRebuildResponse'
+      summary: Rebuild vCenter object cache
+      tags:
+      - vcenters
   /metrics:
     get:
       description: Exposes Prometheus metrics for vctp.