Add vCenter reference cache tables and update related functions

internal/tasks/inventorySnapshots.go

@@ -682,7 +682,7 @@ func (c *CronTask) generateReport(ctx context.Context, tableName string) error {
 	return err
 }
 
-func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTime time.Time, inv *queries.Inventory, hostLookup map[string]vcenter.HostLookup, folderLookup vcenter.FolderLookup, rpLookup map[string]string) (InventorySnapshotRow, error) {
+func snapshotFromVM(ctx context.Context, dbConn *sqlx.DB, vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTime time.Time, inv *queries.Inventory, hostLookup map[string]vcenter.HostLookup, folderLookup vcenter.FolderLookup, rpLookup map[string]string) (InventorySnapshotRow, error) {
 	if vmObject == nil {
 		return InventorySnapshotRow{}, fmt.Errorf("missing VM object")
 	}
@@ -772,23 +772,93 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
 	}
 
 	if row.ResourcePool.String == "" && vmObject.ResourcePool != nil {
+		resourcePoolRef := vmObject.ResourcePool.Value
 		if rpLookup != nil {
-			if rpName, ok := rpLookup[vmObject.ResourcePool.Value]; ok {
-				row.ResourcePool = sql.NullString{String: normalizeResourcePool(rpName), Valid: rpName != ""}
+			if rpName, ok := rpLookup[resourcePoolRef]; ok {
+				rpName = normalizeResourcePool(rpName)
+				row.ResourcePool = sql.NullString{String: rpName, Valid: rpName != ""}
+			}
+		}
+		if row.ResourcePool.String == "" {
+			rpName, err := vc.GetResourcePoolNameByRef(*vmObject.ResourcePool)
+			if err != nil {
+				if vc.Logger != nil {
+					vc.Logger.Warn("failed to resolve resource pool from vcenter", "vm_id", vmObject.Reference().Value, "resource_pool_ref", resourcePoolRef, "error", err)
+				}
+			} else {
+				rpName = normalizeResourcePool(rpName)
+				row.ResourcePool = sql.NullString{String: rpName, Valid: rpName != ""}
+				if rpLookup != nil && rpName != "" {
+					rpLookup[resourcePoolRef] = rpName
+				}
+				if dbConn != nil && rpName != "" {
+					if err := db.UpsertVcenterResourcePoolCache(ctx, dbConn, vc.Vurl, resourcePoolRef, rpName, snapshotTime.Unix()); err != nil && vc.Logger != nil {
+						vc.Logger.Warn("failed to persist resource pool cache", "vcenter", vc.Vurl, "resource_pool_ref", resourcePoolRef, "error", err)
+					}
+				}
 			}
 		}
 	}
 
 	if row.Folder.String == "" {
-		if folderPath, ok := vc.GetVMFolderPathFromLookup(*vmObject, folderLookup); ok {
-			row.Folder = sql.NullString{String: folderPath, Valid: folderPath != ""}
-		} else {
-			// Unable to resolve folder path from lookup; leave empty.
+		folderRef := ""
+		if vmObject.Parent != nil {
+			folderRef = vmObject.Parent.Value
+		}
+		if folderRef != "" && folderLookup != nil {
+			if folderPath, ok := folderLookup[folderRef]; ok {
+				row.Folder = sql.NullString{String: folderPath, Valid: folderPath != ""}
+			}
+		}
+		if row.Folder.String == "" {
+			folderPath, err := vc.GetVMFolderPath(*vmObject)
+			if err != nil {
+				if vc.Logger != nil {
+					vc.Logger.Warn("failed to resolve folder path from vcenter", "vm_id", vmObject.Reference().Value, "error", err)
+				}
+			} else {
+				row.Folder = sql.NullString{String: folderPath, Valid: folderPath != ""}
+				if folderLookup != nil && folderRef != "" && folderPath != "" {
+					folderLookup[folderRef] = folderPath
+				}
+				if dbConn != nil && folderRef != "" && folderPath != "" {
+					if err := db.UpsertVcenterFolderCache(ctx, dbConn, vc.Vurl, folderRef, folderPath, snapshotTime.Unix()); err != nil && vc.Logger != nil {
+						vc.Logger.Warn("failed to persist folder cache", "vcenter", vc.Vurl, "folder_ref", folderRef, "error", err)
+					}
+				}
+			}
 		}
 	}
 
-	if vmObject.Runtime.Host != nil && hostLookup != nil {
-		if lookup, ok := hostLookup[vmObject.Runtime.Host.Value]; ok {
+	hostRef := ""
+	if vmObject.Runtime.Host != nil {
+		hostRef = vmObject.Runtime.Host.Value
+	}
+	if hostRef != "" {
+		lookup, found := vcenter.HostLookup{}, false
+		if hostLookup != nil {
+			lookup, found = hostLookup[hostRef]
+		}
+		if !found {
+			resolved, err := vc.GetHostLookupByRef(*vmObject.Runtime.Host)
+			if err != nil {
+				if vc.Logger != nil {
+					vc.Logger.Warn("failed to resolve host lookup from vcenter", "vm_id", vmObject.Reference().Value, "host_ref", hostRef, "error", err)
+				}
+			} else {
+				lookup = resolved
+				found = true
+				if hostLookup != nil {
+					hostLookup[hostRef] = resolved
+				}
+				if dbConn != nil && (resolved.Cluster != "" || resolved.Datacenter != "") {
+					if err := db.UpsertVcenterHostCache(ctx, dbConn, vc.Vurl, hostRef, resolved.Cluster, resolved.Datacenter, snapshotTime.Unix()); err != nil && vc.Logger != nil {
+						vc.Logger.Warn("failed to persist host cache", "vcenter", vc.Vurl, "host_ref", hostRef, "error", err)
+					}
+				}
+			}
+		}
+		if found {
 			if row.Cluster.String == "" && lookup.Cluster != "" {
 				row.Cluster = sql.NullString{String: lookup.Cluster, Valid: true}
 			}
@@ -801,6 +871,15 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
 	if row.Cluster.String == "" && vmObject.Runtime.Host != nil {
 		if clusterName, err := vc.GetClusterFromHost(vmObject.Runtime.Host); err == nil {
 			row.Cluster = sql.NullString{String: clusterName, Valid: clusterName != ""}
+			if dbConn != nil && hostRef != "" && clusterName != "" {
+				dc := ""
+				if row.Datacenter.Valid {
+					dc = row.Datacenter.String
+				}
+				if err := db.UpsertVcenterHostCache(ctx, dbConn, vc.Vurl, hostRef, clusterName, dc, snapshotTime.Unix()); err != nil && vc.Logger != nil {
+					vc.Logger.Warn("failed to persist host cluster cache", "vcenter", vc.Vurl, "host_ref", hostRef, "error", err)
+				}
+			}
 		} else if vc.Logger != nil {
 			vc.Logger.Warn("failed to resolve cluster from host", "vm_id", vmObject.Reference().Value, "error", err)
 		}
@@ -809,6 +888,15 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
 	if row.Datacenter.String == "" {
 		if dcName, err := vc.GetDatacenterForVM(*vmObject); err == nil {
 			row.Datacenter = sql.NullString{String: dcName, Valid: dcName != ""}
+			if dbConn != nil && hostRef != "" && dcName != "" {
+				clusterName := ""
+				if row.Cluster.Valid {
+					clusterName = row.Cluster.String
+				}
+				if err := db.UpsertVcenterHostCache(ctx, dbConn, vc.Vurl, hostRef, clusterName, dcName, snapshotTime.Unix()); err != nil && vc.Logger != nil {
+					vc.Logger.Warn("failed to persist host datacenter cache", "vcenter", vc.Vurl, "host_ref", hostRef, "error", err)
+				}
+			}
 		}
 	}
 
@@ -958,7 +1046,7 @@ func (c *CronTask) buildPresentSnapshots(ctx context.Context, dbConn *sqlx.DB, v
 			inv = &existingCopy
 		}
 
-		row, err := snapshotFromVM(&vm, vc, startTime, inv, hostLookup, folderLookup, rpLookup)
+		row, err := snapshotFromVM(ctx, dbConn, &vm, vc, startTime, inv, hostLookup, folderLookup, rpLookup)
 		if err != nil {
 			log.Error("unable to build snapshot for VM", "vm_id", vm.Reference().Value, "error", err)
 			continue
@@ -1025,27 +1113,40 @@ func (c *CronTask) initVcenterResources(ctx context.Context, log *slog.Logger, u
 	if err := db.EnsureVmIdentityTables(ctx, c.Database.DB()); err != nil {
 		log.Warn("failed to ensure vm identity tables", "error", err)
 	}
+	if err := db.EnsureVcenterReferenceCacheTables(ctx, c.Database.DB()); err != nil {
+		log.Warn("failed to ensure vcenter reference cache tables", "error", err)
+	}
 
-	hostLookup, err := vc.BuildHostLookup()
+	hostLookup := make(map[string]vcenter.HostLookup)
+	cachedHosts, err := db.LoadVcenterHostCache(ctx, c.Database.DB(), url)
 	if err != nil {
-		log.Warn("failed to build host lookup", "url", url, "error", err)
-		hostLookup = nil
+		log.Warn("failed to load host lookup cache", "url", url, "error", err)
 	} else {
-		log.Debug("built host lookup", "url", url, "hosts", len(hostLookup))
+		for hostRef, entry := range cachedHosts {
+			hostLookup[hostRef] = vcenter.HostLookup{
+				Cluster:    entry.Cluster,
+				Datacenter: entry.Datacenter,
+			}
+		}
+		log.Debug("loaded host lookup cache", "url", url, "hosts", len(hostLookup))
 	}
-	folderLookup, err := vc.BuildFolderPathLookup()
+
+	folderLookup := make(map[string]string)
+	cachedFolders, err := db.LoadVcenterFolderCache(ctx, c.Database.DB(), url)
 	if err != nil {
-		log.Warn("failed to build folder lookup", "url", url, "error", err)
-		folderLookup = nil
+		log.Warn("failed to load folder lookup cache", "url", url, "error", err)
 	} else {
-		log.Debug("built folder lookup", "url", url, "folders", len(folderLookup))
+		folderLookup = cachedFolders
+		log.Debug("loaded folder lookup cache", "url", url, "folders", len(folderLookup))
 	}
-	rpLookup, err := vc.BuildResourcePoolLookup()
+
+	rpLookup := make(map[string]string)
+	cachedPools, err := db.LoadVcenterResourcePoolCache(ctx, c.Database.DB(), url)
 	if err != nil {
-		log.Warn("failed to build resource pool lookup", "url", url, "error", err)
-		rpLookup = nil
+		log.Warn("failed to load resource pool cache", "url", url, "error", err)
 	} else {
-		log.Debug("built resource pool lookup", "url", url, "pools", len(rpLookup))
+		rpLookup = cachedPools
+		log.Debug("loaded resource pool cache", "url", url, "pools", len(rpLookup))
 	}
 
 	res.vms = vms

internal/vcenter/vcenter.go

@@ -429,20 +429,17 @@ func (v *Vcenter) findVmDeletionEvents(ctx context.Context, begin, end time.Time
 			if len(events) == 0 {
 				break
 			}
+			pageCount++
 			processEvents(events)
 			if candidateSet != nil && foundCandidates >= len(candidateSet) {
 				break
 			}
-			pageCount++
 			if pageCount >= maxEventPages {
 				if v.Logger != nil {
 					v.Logger.Warn("vcenter deletion events truncated", "vcenter", v.Vurl, "label", label, "pages", pageCount, "page_size", eventPageSize, "window_start_utc", beginUTC, "window_end_utc", endUTC)
 				}
 				break
 			}
-			if len(events) < int(eventPageSize) {
-				break
-			}
 		}
 		return nil
 	}
@@ -454,6 +451,7 @@ func (v *Vcenter) findVmDeletionEvents(ctx context.Context, begin, end time.Time
 			BeginTime: &beginUTC,
 			EndTime:   &endUTC,
 		},
+		MaxCount:           eventPageSize,
 		DisableFullMessage: &disableFullMessage,
 		EventTypeId: []string{
 			"VmRemovedEvent",
@@ -477,6 +475,7 @@ func (v *Vcenter) findVmDeletionEvents(ctx context.Context, begin, end time.Time
 				BeginTime: &beginUTC,
 				EndTime:   &endUTC,
 			},
+			MaxCount:           eventPageSize,
 			DisableFullMessage: &disableFullMessage,
 		}
 		fc, err := mgr.CreateCollectorForEvents(ctx, fallbackFilter)
@@ -673,31 +672,55 @@ func (v *Vcenter) GetHostSystemObject(hostRef types.ManagedObjectReference) (*mo
 	return &hs, nil
 }
 
+func (v *Vcenter) GetHostLookupByRef(hostRef types.ManagedObjectReference) (HostLookup, error) {
+	lookup := HostLookup{}
+
+	var host mo.HostSystem
+	if err := v.client.RetrieveOne(v.ctx, hostRef, []string{"parent"}, &host); err != nil {
+		return lookup, fmt.Errorf("failed to retrieve host parent: %w", err)
+	}
+
+	if host.Parent != nil && (host.Parent.Type == "ClusterComputeResource" || host.Parent.Type == "ComputeResource") {
+		var moCompute mo.ComputeResource
+		if err := v.client.RetrieveOne(v.ctx, *host.Parent, []string{"name"}, &moCompute); err != nil {
+			if v.Logger != nil {
+				v.Logger.Warn("failed to resolve cluster/compute name from host parent", "host_ref", hostRef.Value, "error", err)
+			}
+		} else {
+			lookup.Cluster = moCompute.Name
+		}
+	}
+
+	entities, err := mo.Ancestors(v.ctx, v.client.Client, v.client.ServiceContent.PropertyCollector, hostRef)
+	if err != nil {
+		if v.Logger != nil {
+			v.Logger.Warn("failed to resolve datacenter from host ancestors", "host_ref", hostRef.Value, "error", err)
+		}
+		return lookup, nil
+	}
+	for _, entity := range entities {
+		if entity.Self.Type != "Datacenter" {
+			continue
+		}
+		lookup.Datacenter = entity.Name
+		break
+	}
+
+	return lookup, nil
+}
+
 // Function to find the cluster or compute resource from a host reference
 func (v *Vcenter) GetClusterFromHost(hostRef *types.ManagedObjectReference) (string, error) {
 	if hostRef == nil {
 		v.Logger.Warn("nil hostRef passed to GetClusterFromHost")
 		return "", nil
 	}
-	var host mo.HostSystem
-	err := v.client.RetrieveOne(v.ctx, *hostRef, []string{"parent"}, &host)
+	lookup, err := v.GetHostLookupByRef(*hostRef)
 	if err != nil {
-		return "", fmt.Errorf("failed to retrieve host parent: %w", err)
+		v.Logger.Error("cant get host lookup", "error", err)
+		return "", err
 	}
-
-	if host.Parent == nil {
-		return "", nil
-	}
-	if host.Parent.Type != "ClusterComputeResource" && host.Parent.Type != "ComputeResource" {
-		return "", nil
-	}
-
-	var moCompute mo.ComputeResource
-	if err := v.client.RetrieveOne(v.ctx, *host.Parent, []string{"name"}, &moCompute); err != nil {
-		return "", fmt.Errorf("failed to retrieve compute resource: %w", err)
-	}
-	v.Logger.Debug("vm host parent compute resource resolved", "name", moCompute.Name, "host_ref", hostRef.Value)
-	return moCompute.Name, nil
+	return lookup.Cluster, nil
 }
 
 // Function to determine the datacenter a VM belongs to
@@ -784,12 +807,19 @@ func (v *Vcenter) FindVMByIDWithDatacenter(vmID string, dcID string) (*mo.Virtua
 	return &vm, nil
 }
 
+func (v *Vcenter) GetResourcePoolNameByRef(resourcePoolRef types.ManagedObjectReference) (string, error) {
+	var pool mo.ResourcePool
+	if err := v.client.RetrieveOne(v.ctx, resourcePoolRef, []string{"name"}, &pool); err != nil {
+		return "", fmt.Errorf("failed to retrieve resource pool: %w", err)
+	}
+	return pool.Name, nil
+}
+
 // Helper function to retrieve the resource pool for the VM
 func (v *Vcenter) GetVmResourcePool(vm mo.VirtualMachine) (string, error) {
 	var resourcePool string
 	if vm.ResourcePool != nil {
-		rp := object.NewResourcePool(v.client.Client, *vm.ResourcePool)
-		rpName, err := rp.ObjectName(v.ctx)
+		rpName, err := v.GetResourcePoolNameByRef(*vm.ResourcePool)
 		if err != nil {
 			v.Logger.Error("failed to get resource pool name", "error", err)
 			return resourcePool, err