Add vCenter cache rebuild functionality and related API endpoint

db/helpers.go

@@ -1161,6 +1161,35 @@ CREATE TABLE IF NOT EXISTS vcenter_host_cache (
 	})
 }
 
+// ClearVcenterReferenceCache removes cached folder/resource-pool/host references for a vCenter.
+func ClearVcenterReferenceCache(ctx context.Context, dbConn *sqlx.DB, vcenter string) error {
+	vcenter = strings.TrimSpace(vcenter)
+	if vcenter == "" {
+		return nil
+	}
+	if err := EnsureVcenterReferenceCacheTables(ctx, dbConn); err != nil {
+		return err
+	}
+
+	tx, err := dbConn.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	tables := []string{
+		"vcenter_folder_cache",
+		"vcenter_resource_pool_cache",
+		"vcenter_host_cache",
+	}
+	for _, tableName := range tables {
+		query := tx.Rebind(fmt.Sprintf(`DELETE FROM %s WHERE "Vcenter" = ?`, tableName))
+		if _, err := tx.ExecContext(ctx, query, vcenter); err != nil {
+			_ = tx.Rollback()
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
 func LoadVcenterFolderCache(ctx context.Context, dbConn *sqlx.DB, vcenter string) (map[string]string, error) {
 	cache := make(map[string]string)
 	vcenter = strings.TrimSpace(vcenter)

db/helpers_cache_and_index_test.go

@@ -310,6 +310,52 @@ VALUES (?,?,?,?,?,?,?,?,?,?)
 	}
 }
 
+func TestClearVcenterReferenceCache(t *testing.T) {
+	ctx := context.Background()
+	dbConn := newTestSQLiteDB(t)
+
+	if err := EnsureVcenterReferenceCacheTables(ctx, dbConn); err != nil {
+		t.Fatalf("failed to ensure vcenter reference cache tables: %v", err)
+	}
+	if err := UpsertVcenterFolderCache(ctx, dbConn, "vc-a", "group-v123", "/Datacenters/DC1/vm/Prod", 1000); err != nil {
+		t.Fatalf("failed to upsert folder cache: %v", err)
+	}
+	if err := UpsertVcenterResourcePoolCache(ctx, dbConn, "vc-a", "resgroup-1", "Gold", 1000); err != nil {
+		t.Fatalf("failed to upsert resource pool cache: %v", err)
+	}
+	if err := UpsertVcenterHostCache(ctx, dbConn, "vc-a", "host-123", "Cluster-1", "DC1", 1000); err != nil {
+		t.Fatalf("failed to upsert host cache: %v", err)
+	}
+
+	if err := ClearVcenterReferenceCache(ctx, dbConn, "vc-a"); err != nil {
+		t.Fatalf("failed to clear vcenter reference cache: %v", err)
+	}
+
+	var folderCount int
+	if err := dbConn.Get(&folderCount, `SELECT COUNT(1) FROM vcenter_folder_cache WHERE "Vcenter" = ?`, "vc-a"); err != nil {
+		t.Fatalf("failed to count folder cache rows: %v", err)
+	}
+	if folderCount != 0 {
+		t.Fatalf("expected 0 folder cache rows after clear, got %d", folderCount)
+	}
+
+	var poolCount int
+	if err := dbConn.Get(&poolCount, `SELECT COUNT(1) FROM vcenter_resource_pool_cache WHERE "Vcenter" = ?`, "vc-a"); err != nil {
+		t.Fatalf("failed to count resource pool cache rows: %v", err)
+	}
+	if poolCount != 0 {
+		t.Fatalf("expected 0 resource pool cache rows after clear, got %d", poolCount)
+	}
+
+	var hostCount int
+	if err := dbConn.Get(&hostCount, `SELECT COUNT(1) FROM vcenter_host_cache WHERE "Vcenter" = ?`, "vc-a"); err != nil {
+		t.Fatalf("failed to count host cache rows: %v", err)
+	}
+	if hostCount != 0 {
+		t.Fatalf("expected 0 host cache rows after clear, got %d", hostCount)
+	}
+}
+
 func TestFetchVmLifecycleIgnoresStaleDeletionFromHourlyCache(t *testing.T) {
 	ctx := context.Background()
 	dbConn := newTestSQLiteDB(t)

internal/report/snapshots.go

@@ -749,6 +749,12 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
 	}
 
 	if isDailySummary || isMonthlySummary {
+		reportHeaders := make([]string, 0, len(specs))
+		for _, spec := range specs {
+			reportHeaders = append(reportHeaders, spec.Name)
+		}
+		addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName)
+
 		meta := reportMetadata{
 			TableName:   tableName,
 			ReportType:  reportTypeFromTable(tableName),
@@ -883,6 +889,144 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
 	}
 }
 
+type summaryPivotSpec struct {
+	Title       string
+	TitleCell   string
+	PivotName   string
+	PivotRange  string
+	RowFields   []string
+	DataField   string
+	DataName    string
+	DataSummary string
+}
+
+func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string) {
+	if logger == nil {
+		logger = slog.Default()
+	}
+	const summarySheet = "Summary"
+	if _, err := xlsx.NewSheet(summarySheet); err != nil {
+		logger.Warn("failed to create summary worksheet", "table", tableName, "error", err)
+		return
+	}
+
+	if rowCount <= 0 {
+		xlsx.SetCellValue(summarySheet, "A1", "Summary")
+		xlsx.SetCellValue(summarySheet, "A3", "No data rows were available to build pivot tables.")
+		if err := SetColAutoWidth(xlsx, summarySheet); err != nil {
+			logger.Warn("failed to size summary worksheet columns", "table", tableName, "error", err)
+		}
+		return
+	}
+	if len(headers) == 0 {
+		logger.Warn("summary worksheet skipped due to empty headers", "table", tableName)
+		return
+	}
+
+	endCell, err := excelize.CoordinatesToCellName(len(headers), rowCount+1)
+	if err != nil {
+		logger.Warn("summary worksheet skipped due to invalid data range", "table", tableName, "error", err)
+		return
+	}
+	dataRange := fmt.Sprintf("%s!A1:%s", quoteSheetName(dataSheet), endCell)
+	lowerToHeader := make(map[string]string, len(headers))
+	for _, header := range headers {
+		lowerToHeader[strings.ToLower(strings.TrimSpace(header))] = header
+	}
+	resolveField := func(name string) (string, bool) {
+		header, ok := lowerToHeader[strings.ToLower(strings.TrimSpace(name))]
+		return header, ok
+	}
+
+	specs := []summaryPivotSpec{
+		{
+			Title:       "Sum of Avg vCPUs",
+			TitleCell:   "A1",
+			PivotName:   "PivotAvgVcpu",
+			PivotRange:  fmt.Sprintf("%s!A3:H1000", quoteSheetName(summarySheet)),
+			RowFields:   []string{"Datacenter", "ResourcePool"},
+			DataField:   "AvgVcpuCount",
+			DataName:    "Sum of Avg vCPUs",
+			DataSummary: "Sum",
+		},
+		{
+			Title:       "Sum of Avg RAM",
+			TitleCell:   "J1",
+			PivotName:   "PivotAvgRam",
+			PivotRange:  fmt.Sprintf("%s!J3:P1000", quoteSheetName(summarySheet)),
+			RowFields:   []string{"Datacenter"},
+			DataField:   "AvgRamGB",
+			DataName:    "Sum of Avg RAM",
+			DataSummary: "Sum",
+		},
+		{
+			Title:       "Sum of prorated VM count",
+			TitleCell:   "A1003",
+			PivotName:   "PivotProratedVmCount",
+			PivotRange:  fmt.Sprintf("%s!A1005:H2002", quoteSheetName(summarySheet)),
+			RowFields:   []string{"Datacenter"},
+			DataField:   "AvgIsPresent",
+			DataName:    "Sum of prorated VM count",
+			DataSummary: "Sum",
+		},
+		{
+			Title:       "Count of VM Name",
+			TitleCell:   "J1003",
+			PivotName:   "PivotVmNameCount",
+			PivotRange:  fmt.Sprintf("%s!J1005:P2002", quoteSheetName(summarySheet)),
+			RowFields:   []string{"Datacenter"},
+			DataField:   "Name",
+			DataName:    "Count of VM Name",
+			DataSummary: "Count",
+		},
+	}
+
+	for _, spec := range specs {
+		xlsx.SetCellValue(summarySheet, spec.TitleCell, spec.Title)
+
+		rows := make([]excelize.PivotTableField, 0, len(spec.RowFields))
+		missingField := false
+		for _, rowField := range spec.RowFields {
+			resolved, ok := resolveField(rowField)
+			if !ok {
+				logger.Warn("summary pivot skipped: missing row field", "table", tableName, "pivot", spec.PivotName, "field", rowField)
+				missingField = true
+				break
+			}
+			rows = append(rows, excelize.PivotTableField{Data: resolved})
+		}
+		if missingField {
+			continue
+		}
+		dataField, ok := resolveField(spec.DataField)
+		if !ok {
+			logger.Warn("summary pivot skipped: missing data field", "table", tableName, "pivot", spec.PivotName, "field", spec.DataField)
+			continue
+		}
+
+		if err := xlsx.AddPivotTable(&excelize.PivotTableOptions{
+			Name:                spec.PivotName,
+			DataRange:           dataRange,
+			PivotTableRange:     spec.PivotRange,
+			Rows:                rows,
+			Data:                []excelize.PivotTableField{{Data: dataField, Name: spec.DataName, Subtotal: spec.DataSummary}},
+			RowGrandTotals:      true,
+			ColGrandTotals:      true,
+			ShowDrill:           true,
+			ShowRowHeaders:      true,
+			ShowColHeaders:      true,
+			ShowLastColumn:      true,
+			PivotTableStyleName: "PivotStyleLight16",
+		}); err != nil {
+			logger.Warn("failed to add summary pivot table", "table", tableName, "pivot", spec.PivotName, "error", err)
+		}
+	}
+
+	if err := SetColAutoWidth(xlsx, summarySheet); err != nil {
+		logger.Warn("failed to size summary worksheet columns", "table", tableName, "error", err)
+	}
+}
+
 func tableColumns(ctx context.Context, dbConn *sqlx.DB, tableName string) ([]string, error) {
 	driver := strings.ToLower(dbConn.DriverName())
 	switch driver {
@@ -1464,29 +1608,29 @@ FROM diag, agg_diag
 `, vmKeyExpr, overlapExpr, selected.TableName, templateExclusionFilter(), vmKeyExpr, prevTableName, templateExclusionFilter(), missingOverlapExpr, aggOverlapExpr)
 		query = dbConn.Rebind(query)
 		var row struct {
-			VmCount          int64   `db:"vm_count"`
+			VmCount           int64   `db:"vm_count"`
-			VcpuTotal        int64   `db:"vcpu_total"`
+			VcpuTotal         int64   `db:"vcpu_total"`
-			RamTotal         int64   `db:"ram_total"`
+			RamTotal          int64   `db:"ram_total"`
-			PresenceRatio    float64 `db:"presence_ratio"`
+			PresenceRatio     float64 `db:"presence_ratio"`
-			TinTotal         float64 `db:"tin_total"`
+			TinTotal          float64 `db:"tin_total"`
-			BronzeTotal      float64 `db:"bronze_total"`
+			BronzeTotal       float64 `db:"bronze_total"`
-			SilverTotal      float64 `db:"silver_total"`
+			SilverTotal       float64 `db:"silver_total"`
-			GoldTotal        float64 `db:"gold_total"`
+			GoldTotal         float64 `db:"gold_total"`
-			RowCount         int64   `db:"row_count"`
+			RowCount          int64   `db:"row_count"`
-			DistinctKeys     int64   `db:"distinct_keys"`
+			DistinctKeys      int64   `db:"distinct_keys"`
-			UnknownKeys      int64   `db:"unknown_keys"`
+			UnknownKeys       int64   `db:"unknown_keys"`
-			MissingVmID      int64   `db:"missing_vm_id"`
+			MissingVmID       int64   `db:"missing_vm_id"`
-			MissingVmUUID    int64   `db:"missing_vm_uuid"`
+			MissingVmUUID     int64   `db:"missing_vm_uuid"`
-			MissingName      int64   `db:"missing_name"`
+			MissingName       int64   `db:"missing_name"`
-			PresenceOverOne  int64   `db:"presence_over_one"`
+			PresenceOverOne   int64   `db:"presence_over_one"`
-			PresenceUnderZero int64  `db:"presence_under_zero"`
+			PresenceUnderZero int64   `db:"presence_under_zero"`
-			BasePresenceSum  float64 `db:"base_presence_sum"`
+			BasePresenceSum   float64 `db:"base_presence_sum"`
-			AggCount         int64   `db:"agg_count"`
+			AggCount          int64   `db:"agg_count"`
-			MissingCreation  int64   `db:"missing_creation"`
+			MissingCreation   int64   `db:"missing_creation"`
-			MissingDeletion  int64   `db:"missing_deletion"`
+			MissingDeletion   int64   `db:"missing_deletion"`
-			CreatedInInterval int64  `db:"created_in_interval"`
+			CreatedInInterval int64   `db:"created_in_interval"`
-			DeletedInInterval int64  `db:"deleted_in_interval"`
+			DeletedInInterval int64   `db:"deleted_in_interval"`
-			PartialPresence  int64   `db:"partial_presence"`
+			PartialPresence   int64   `db:"partial_presence"`
 		}
 		overlapArgs := []interface{}{
 			hourEndUnix, hourEndUnix,

internal/vcenter/vcenter.go

@@ -874,7 +874,7 @@ func (v *Vcenter) BuildResourcePoolLookup() (map[string]string, error) {
 
 // Helper function to retrieve the full folder path for the VM
 func (v *Vcenter) GetVMFolderPath(vm mo.VirtualMachine) (string, error) {
-	v.Logger.Debug("Commencing vm folder path search")
+	v.Logger.Debug("commencing vm folder path search", "vcenter", v.Vurl, "vm_id", vm.Reference().Value)
 
 	entities, err := mo.Ancestors(v.ctx, v.client.Client, v.client.ServiceContent.PropertyCollector, vm.Reference())
 	if err != nil {

server/handler/vcenterCacheRebuild.go

@@ -0,0 +1,162 @@
+package handler
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+	"vctp/db"
+	"vctp/internal/vcenter"
+	"vctp/server/models"
+)
+
+// VcenterCacheRebuild force-regenerates cached vCenter reference data in the database.
+// @Summary Rebuild vCenter object cache
+// @Description Rebuilds cached folder/resource-pool/host(cluster+datacenter) references from vCenter and rewrites the database cache tables.
+// @Tags vcenters
+// @Produce json
+// @Param vcenter query string false "Optional single vCenter URL to rebuild; defaults to all configured vCenters"
+// @Success 200 {object} models.VcenterCacheRebuildResponse "Cache rebuild summary"
+// @Failure 400 {object} models.ErrorResponse "Invalid request"
+// @Failure 405 {object} models.ErrorResponse "Method not allowed"
+// @Failure 500 {object} models.VcenterCacheRebuildResponse "All rebuild attempts failed"
+// @Router /api/vcenters/cache/rebuild [post]
+func (h *Handler) VcenterCacheRebuild(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		writeJSONError(w, http.StatusMethodNotAllowed, "method not allowed")
+		return
+	}
+
+	ctx := r.Context()
+	h.Settings.ReadYMLSettings()
+
+	requested := strings.TrimSpace(r.URL.Query().Get("vcenter"))
+	configured := h.Settings.Values.Settings.VcenterAddresses
+	targets := make([]string, 0, len(configured))
+	for _, raw := range configured {
+		vcURL := strings.TrimSpace(raw)
+		if vcURL == "" {
+			continue
+		}
+		if requested != "" && vcURL != requested {
+			continue
+		}
+		targets = append(targets, vcURL)
+	}
+
+	if requested != "" && len(targets) == 0 {
+		writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("requested vcenter is not configured: %s", requested))
+		return
+	}
+	if len(targets) == 0 {
+		writeJSONError(w, http.StatusBadRequest, "no vcenter addresses configured")
+		return
+	}
+
+	if err := db.EnsureVcenterReferenceCacheTables(ctx, h.Database.DB()); err != nil {
+		h.Logger.Error("failed to ensure vcenter cache tables", "error", err)
+		writeJSONError(w, http.StatusInternalServerError, "failed to ensure vcenter cache tables")
+		return
+	}
+
+	resp := models.VcenterCacheRebuildResponse{
+		Status:  "OK",
+		Total:   len(targets),
+		Results: make([]models.VcenterCacheRebuildResult, 0, len(targets)),
+	}
+
+	for _, vcURL := range targets {
+		start := time.Now()
+		result := models.VcenterCacheRebuildResult{
+			Vcenter: vcURL,
+		}
+
+		folderEntries, poolEntries, hostEntries, err := h.rebuildOneVcenterCache(ctx, vcURL)
+		result.DurationSeconds = math.Round(time.Since(start).Seconds()*1000) / 1000
+		if err != nil {
+			result.Error = err.Error()
+			resp.Failed++
+			h.Logger.Warn("vcenter cache rebuild failed", "vcenter", vcURL, "error", err)
+		} else {
+			result.FolderEntries = folderEntries
+			result.ResourcePoolEntries = poolEntries
+			result.HostEntries = hostEntries
+			resp.Succeeded++
+			h.Logger.Info("vcenter cache rebuild completed", "vcenter", vcURL, "folder_entries", folderEntries, "resource_pool_entries", poolEntries, "host_entries", hostEntries, "duration", time.Since(start))
+		}
+		resp.Results = append(resp.Results, result)
+	}
+
+	switch {
+	case resp.Failed == 0:
+		resp.Status = "OK"
+		writeJSON(w, http.StatusOK, resp)
+	case resp.Succeeded == 0:
+		resp.Status = "ERROR"
+		writeJSON(w, http.StatusInternalServerError, resp)
+	default:
+		resp.Status = "PARTIAL"
+		writeJSON(w, http.StatusOK, resp)
+	}
+}
+
+func (h *Handler) rebuildOneVcenterCache(ctx context.Context, vcURL string) (int, int, int, error) {
+	vc := vcenter.New(h.Logger, h.VcCreds)
+	if err := vc.Login(vcURL); err != nil {
+		return 0, 0, 0, fmt.Errorf("unable to connect to vcenter: %w", err)
+	}
+	defer func() {
+		logoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := vc.Logout(logoutCtx); err != nil {
+			h.Logger.Warn("vcenter cache rebuild logout failed", "vcenter", vcURL, "error", err)
+		}
+	}()
+
+	folderLookup, err := vc.BuildFolderPathLookup()
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("failed to build folder cache from vcenter: %w", err)
+	}
+	resourcePools, err := vc.BuildResourcePoolLookup()
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("failed to build resource pool cache from vcenter: %w", err)
+	}
+	hostLookup, err := vc.BuildHostLookup()
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("failed to build host cache from vcenter: %w", err)
+	}
+
+	dbConn := h.Database.DB()
+	if err := db.ClearVcenterReferenceCache(ctx, dbConn, vcURL); err != nil {
+		return 0, 0, 0, fmt.Errorf("failed to clear existing vcenter cache rows: %w", err)
+	}
+	lastSeen := time.Now().Unix()
+
+	folderCount := 0
+	for folderRef, folderPath := range folderLookup {
+		if err := db.UpsertVcenterFolderCache(ctx, dbConn, vcURL, folderRef, folderPath, lastSeen); err != nil {
+			return 0, 0, 0, fmt.Errorf("failed to persist folder cache: %w", err)
+		}
+		folderCount++
+	}
+
+	poolCount := 0
+	for poolRef, poolName := range resourcePools {
+		if err := db.UpsertVcenterResourcePoolCache(ctx, dbConn, vcURL, poolRef, poolName, lastSeen); err != nil {
+			return 0, 0, 0, fmt.Errorf("failed to persist resource pool cache: %w", err)
+		}
+		poolCount++
+	}
+
+	hostCount := 0
+	for hostRef, entry := range hostLookup {
+		if err := db.UpsertVcenterHostCache(ctx, dbConn, vcURL, hostRef, entry.Cluster, entry.Datacenter, lastSeen); err != nil {
+			return 0, 0, 0, fmt.Errorf("failed to persist host cache: %w", err)
+		}
+		hostCount++
+	}
+
+	return folderCount, poolCount, hostCount, nil
+}

server/models/api_responses.go

@@ -59,3 +59,22 @@ type SnapshotRepairSuiteResponse struct {
 	MonthlyRefined string `json:"monthly_refined"`
 	MonthlyFailed  string `json:"monthly_failed"`
 }
+
+// VcenterCacheRebuildResult describes rebuild results for a single vCenter.
+type VcenterCacheRebuildResult struct {
+	Vcenter             string  `json:"vcenter"`
+	FolderEntries       int     `json:"folder_entries"`
+	ResourcePoolEntries int     `json:"resource_pool_entries"`
+	HostEntries         int     `json:"host_entries"`
+	DurationSeconds     float64 `json:"duration_seconds"`
+	Error               string  `json:"error,omitempty"`
+}
+
+// VcenterCacheRebuildResponse describes the vCenter object cache rebuild response.
+type VcenterCacheRebuildResponse struct {
+	Status    string                      `json:"status"`
+	Total     int                         `json:"total"`
+	Succeeded int                         `json:"succeeded"`
+	Failed    int                         `json:"failed"`
+	Results   []VcenterCacheRebuildResult `json:"results"`
+}

server/router/router.go

@@ -74,6 +74,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
 	mux.HandleFunc("/vcenters/totals", h.VcenterTotals)
 	mux.HandleFunc("/vcenters/totals/daily", h.VcenterTotalsDaily)
 	mux.HandleFunc("/vcenters/totals/hourly", h.VcenterTotalsHourlyDetailed)
+	mux.HandleFunc("/api/vcenters/cache/rebuild", h.VcenterCacheRebuild)
 	mux.HandleFunc("/metrics", h.Metrics)
 
 	mux.HandleFunc("/snapshots/hourly", h.SnapshotHourlyList)