add vm inventory update endpoint

db/migrations/20240913043145_extend_events.sql

@@ -10,8 +10,8 @@ ALTER TABLE "Events" ADD COLUMN VmName TEXT;
 -- +goose Down
 -- +goose StatementBegin
 ALTER TABLE "Events" DROP COLUMN VmName;
-ALTER TABLE "Updates" DROP COLUMN ComputeResourceId;
-ALTER TABLE "Updates" DROP COLUMN DatacenterId;
+ALTER TABLE "Events" DROP COLUMN ComputeResourceId;
+ALTER TABLE "Events" DROP COLUMN DatacenterId;
 ALTER TABLE "Events" RENAME COLUMN ComputeResourceName to ComputeResource;
 ALTER TABLE "Events" RENAME COLUMN DatacenterName to Datacenter;
 -- +goose StatementEnd

db/migrations/20240930031506_add_table.sql

@@ -0,0 +1,18 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS "InventoryHistory" (
+    "Hid" INTEGER PRIMARY KEY AUTOINCREMENT,
+    "InventoryId" INTEGER,
+    "ReportDate" INTEGER,    
+    "UpdateTime" INTEGER,
+    "PreviousVcpus" INTEGER,
+    "PreviousRam" INTEGER,
+    "PreviousResourcePool" TEXT,
+    "PreviousProvisionedDisk" REAL
+)
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE "InventoryHistory";
+-- +goose StatementEnd

db/queries/models.go

@@ -49,6 +49,17 @@ type Inventory struct {
 	VmUuid          sql.NullString
 }
 
+type InventoryHistory struct {
+	Hid                     int64
+	InventoryId             sql.NullInt64
+	ReportDate              sql.NullInt64
+	UpdateTime              sql.NullInt64
+	PreviousVcpus           sql.NullInt64
+	PreviousRam             sql.NullInt64
+	PreviousResourcePool    sql.NullString
+	PreviousProvisionedDisk sql.NullFloat64
+}
+
 type Updates struct {
 	Uid                int64
 	InventoryId        sql.NullInt64

db/queries/query.sql

@@ -38,6 +38,11 @@ INSERT INTO "Inventory" (
 )
 RETURNING *;
 
+-- name: InventoryUpdate :exec
+UPDATE "Inventory"
+SET "VmUuid" = sqlc.arg('uuid'), "SrmPlaceholder" = sqlc.arg('srmPlaceholder')
+WHERE "Iid" = sqlc.arg('iid');
+
 -- name: InventoryMarkDeleted :exec
 UPDATE "Inventory"
 SET "DeletionTime" = sqlc.arg('deletionTime')
@@ -91,4 +96,12 @@ ORDER BY "EventTime";
 -- name: UpdateEventsProcessed :exec
 UPDATE "Events"
 SET "Processed" = 1
-WHERE "Eid" = sqlc.arg('eid');
+WHERE "Eid" = sqlc.arg('eid');
+
+-- name: CreateInventoryHistory :one
+INSERT INTO "InventoryHistory" (
+    "InventoryId", "ReportDate", "UpdateTime", "PreviousVcpus", "PreviousRam", "PreviousResourcePool", "PreviousProvisionedDisk"
+) VALUES(
+    ?, ?, ?, ?, ?, ?, ?
+)
+RETURNING *;

db/queries/query.sql.go

@@ -165,6 +165,49 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams
 	return i, err
 }
 
+const createInventoryHistory = `-- name: CreateInventoryHistory :one
+INSERT INTO "InventoryHistory" (
+    "InventoryId", "ReportDate", "UpdateTime", "PreviousVcpus", "PreviousRam", "PreviousResourcePool", "PreviousProvisionedDisk"
+) VALUES(
+    ?, ?, ?, ?, ?, ?, ?
+)
+RETURNING Hid, InventoryId, ReportDate, UpdateTime, PreviousVcpus, PreviousRam, PreviousResourcePool, PreviousProvisionedDisk
+`
+
+type CreateInventoryHistoryParams struct {
+	InventoryId             sql.NullInt64
+	ReportDate              sql.NullInt64
+	UpdateTime              sql.NullInt64
+	PreviousVcpus           sql.NullInt64
+	PreviousRam             sql.NullInt64
+	PreviousResourcePool    sql.NullString
+	PreviousProvisionedDisk sql.NullFloat64
+}
+
+func (q *Queries) CreateInventoryHistory(ctx context.Context, arg CreateInventoryHistoryParams) (InventoryHistory, error) {
+	row := q.db.QueryRowContext(ctx, createInventoryHistory,
+		arg.InventoryId,
+		arg.ReportDate,
+		arg.UpdateTime,
+		arg.PreviousVcpus,
+		arg.PreviousRam,
+		arg.PreviousResourcePool,
+		arg.PreviousProvisionedDisk,
+	)
+	var i InventoryHistory
+	err := row.Scan(
+		&i.Hid,
+		&i.InventoryId,
+		&i.ReportDate,
+		&i.UpdateTime,
+		&i.PreviousVcpus,
+		&i.PreviousRam,
+		&i.PreviousResourcePool,
+		&i.PreviousProvisionedDisk,
+	)
+	return i, err
+}
+
 const createUpdate = `-- name: CreateUpdate :one
 INSERT INTO "Updates" (
     "InventoryId", "EventKey", "EventId", "UpdateTime", "UpdateType", "NewVcpus", "NewRam", "NewResourcePool", "NewProvisionedDisk", "UserName"
@@ -606,6 +649,23 @@ func (q *Queries) InventoryMarkDeleted(ctx context.Context, arg InventoryMarkDel
 	return err
 }
 
+const inventoryUpdate = `-- name: InventoryUpdate :exec
+UPDATE "Inventory"
+SET "VmUuid" = ?1, "SrmPlaceholder" = ?2
+WHERE "Iid" = ?3
+`
+
+type InventoryUpdateParams struct {
+	Uuid           sql.NullString
+	SrmPlaceholder interface{}
+	Iid            int64
+}
+
+func (q *Queries) InventoryUpdate(ctx context.Context, arg InventoryUpdateParams) error {
+	_, err := q.db.ExecContext(ctx, inventoryUpdate, arg.Uuid, arg.SrmPlaceholder, arg.Iid)
+	return err
+}
+
 const listEvents = `-- name: ListEvents :many
 SELECT Eid, CloudId, Source, EventTime, ChainId, VmId, EventKey, DatacenterName, ComputeResourceName, UserName, Processed, DatacenterId, ComputeResourceId, VmName, EventType FROM "Events"
 ORDER BY "EventTime"

internal/tasks/monitorVcenter.go

@@ -114,16 +114,16 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 
 	c.Logger.Debug("found VM")
 
-	if vmObject.Name == "DBRaaS_testVMTemplate" {
-		c.Logger.Debug("Found problematic VM")
-		//prettyPrint(vmObject)
-	}
-
-	srmPlaceholder = "FALSE" // Default assumption
-	//prettyPrint(vmObject)
+	/*
+		if vmObject.Name == "DBRaaS_testVMTemplate" {
+			c.Logger.Debug("Found problematic VM")
+			//prettyPrint(vmObject)
+		}
+	*/
 
 	// calculate VM properties we want to store
 	if vmObject.Config != nil {
+		// Skip any template VMs
 		if vmObject.Config.Template {
 			c.Logger.Debug("Not adding templates to inventory")
 			return nil
@@ -133,6 +133,7 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 
 		numRam = vmObject.Config.Hardware.MemoryMB
 		numVcpus = vmObject.Config.Hardware.NumCPU
+		srmPlaceholder = "FALSE" // Default assumption
 
 		// Calculate creation date
 		if vmObject.Config.CreateDate.IsZero() {
@@ -156,16 +157,19 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 				}
 
 				totalDiskBytes += disk.CapacityInBytes
-				//totalDiskGB += float64(disk.CapacityInBytes / 1024 / 1024 / 1024) // Convert from bytes to GB
 			}
 		}
 		totalDiskGB = float64(totalDiskBytes / 1024 / 1024 / 1024)
 		c.Logger.Debug("Converted total disk size", "bytes", totalDiskBytes, "GB", totalDiskGB)
 
 		// Determine if the VM is a normal VM or an SRM placeholder
-		if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.Type == "com.vmware.vcDr" {
-			c.Logger.Debug("VM ManagedBy indicates managed by SRM")
-			srmPlaceholder = "TRUE"
+		if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" {
+			if vmObject.Config.ManagedBy.Type == "placeholderVm" {
+				c.Logger.Debug("VM is a placeholder")
+				srmPlaceholder = "TRUE"
+			} else {
+				c.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObject.Config.ManagedBy)
+			}
 		}
 
 		// Retrieve the full folder path of the VM
@@ -179,10 +183,10 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 
 		foundVmConfig = true
 	} else {
-		c.Logger.Error("Empty VM config")
+		c.Logger.Warn("Empty VM config")
 	}
 
-	c.Logger.Debug("VM has runtime data", "power_state", vmObject.Runtime.PowerState)
+	//c.Logger.Debug("VM has runtime data", "power_state", vmObject.Runtime.PowerState)
 	if vmObject.Runtime.PowerState == "poweredOff" {
 		poweredOn = "FALSE"
 	} else {
@@ -241,7 +245,6 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 	}
 
 	return nil
-
 }
 
 // prettyPrint comes from https://gist.github.com/sfate/9d45f6c5405dc4c9bf63bf95fe6d1a7c

internal/tasks/processEvents.go

@@ -94,9 +94,13 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
 				c.Logger.Debug("Converted total disk size", "bytes", totalDiskBytes, "GB", totalDiskGB)
 
 				// Determine if the VM is a normal VM or an SRM placeholder
-				if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.Type == "com.vmware.vcDr" {
-					c.Logger.Debug("VM ManagedBy indicates managed by SRM")
-					srmPlaceholder = "TRUE"
+				if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" {
+					if vmObject.Config.ManagedBy.Type == "placeholderVm" {
+						c.Logger.Debug("VM is a placeholder")
+						srmPlaceholder = "TRUE"
+					} else {
+						c.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObject.Config.ManagedBy)
+					}
 				}
 
 				if vmObject.Config.Template {

main.go

@@ -209,7 +209,7 @@ func main() {
 	c.Start()
 
 	// Start server
-	r := router.New(logger, database, buildTime, sha1ver, runtime.Version(), &creds, a)
+	r := router.New(logger, database, buildTime, sha1ver, runtime.Version(), &creds, a, s)
 	svr := server.New(
 		logger,
 		c,

server/handler/handler.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"vctp/db"
 	"vctp/internal/secrets"
+	"vctp/internal/settings"
 	"vctp/internal/vcenter"
 
 	"github.com/a-h/templ"
@@ -20,6 +21,7 @@ type Handler struct {
 	GoVersion string
 	VcCreds   *vcenter.VcenterLogin
 	Secret    *secrets.Secrets
+	Settings  *settings.Settings
 }
 
 func (h *Handler) html(ctx context.Context, w http.ResponseWriter, status int, t templ.Component) {

server/handler/vmUpdateDetails.go

@@ -0,0 +1,114 @@
+package handler
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/http"
+	"vctp/db/queries"
+	"vctp/internal/vcenter"
+)
+
+// VmUpdate receives the CloudEvent for a VM modification or move
+func (h *Handler) VmUpdateDetails(w http.ResponseWriter, r *http.Request) {
+	var matchFound bool
+	var inventoryId int64
+	var srmPlaceholder string
+	var vmUuid string
+	var dbUuid string
+
+	ctx := context.Background()
+
+	// reload settings in case vcenter list has changed
+	h.Settings.ReadYMLSettings()
+
+	for _, url := range h.Settings.Values.Settings.VcenterAddresses {
+		h.Logger.Debug("connecting to vcenter", "url", url)
+		vc := vcenter.New(h.Logger, h.VcCreds)
+		vc.Login(url)
+
+		// Get list of VMs from vcenter
+		vms, err := vc.GetAllVmReferences()
+
+		// Get list of VMs from inventory table
+		h.Logger.Debug("Querying inventory table")
+		results, err := h.Database.Queries().GetInventoryByVcenter(ctx, url)
+		if err != nil {
+			h.Logger.Error("Unable to query inventory table", "error", err)
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprintf(w, "Unable to query inventory table %s\n", err)
+		}
+
+		if len(results) == 0 {
+			h.Logger.Error("Empty inventory results")
+		}
+
+		// Iterate VMs from vcenter and see if they were in the database
+		for _, vm := range vms {
+			matchFound = false
+			inventoryId = 0
+			srmPlaceholder = "FALSE" // Default assumption
+			vmUuid = ""
+
+			for _, dbvm := range results {
+				if dbvm.VmId.String == vm.Reference().Value {
+					h.Logger.Debug("Found VM in database", "vm_name", dbvm.Name, "id", dbvm.VmId.String)
+					matchFound = true
+					inventoryId = dbvm.Iid
+					dbUuid = dbvm.VmUuid.String
+					break
+				}
+			}
+
+			if matchFound {
+				//h.Logger.Debug("Need to update VM in inventory table", "MoRef", vm.Reference())
+				vmObj, err := vc.ConvertObjToMoVM(vm)
+				if err != nil {
+					h.Logger.Error("Received error getting vm managedobject", "error", err)
+					continue
+				}
+
+				if vmObj.Config != nil {
+					vmUuid = vmObj.Config.Uuid
+
+					// Determine if the VM is a normal VM or an SRM placeholder
+					if vmObj.Config.ManagedBy != nil && vmObj.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" {
+						if vmObj.Config.ManagedBy.Type == "placeholderVm" {
+							h.Logger.Debug("VM is a placeholder")
+							srmPlaceholder = "TRUE"
+						} else {
+							h.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObj.Config.ManagedBy)
+						}
+					}
+
+					if srmPlaceholder == "TRUE" || vmUuid != dbUuid {
+						h.Logger.Debug("Need to update vm", "name", vmObj.Name, "srm_placeholder", srmPlaceholder, "uuid", vmUuid)
+						params := queries.InventoryUpdateParams{
+							Iid:            inventoryId,
+							SrmPlaceholder: srmPlaceholder,
+							Uuid:           sql.NullString{String: vmUuid, Valid: vmUuid != ""},
+						}
+
+						h.Logger.Debug("database params", "params", params)
+						err := h.Database.Queries().InventoryUpdate(context.Background(), params)
+
+						if err != nil {
+							h.Logger.Error("Error received updating inventory for VM", "name", vmObj.Name, "error", err)
+						}
+
+					}
+
+				} else {
+					h.Logger.Warn("VM no longer present in vcenter or missing config values", "MoRef", vm.Reference())
+				}
+
+			}
+
+		}
+	}
+
+	h.Logger.Debug("Processed vm update successfully")
+	w.WriteHeader(http.StatusOK)
+	// TODO - return some JSON
+	fmt.Fprintf(w, "Processed vm update successfully")
+}

server/router/router.go

@@ -7,12 +7,13 @@ import (
 	"vctp/db"
 	"vctp/dist"
 	"vctp/internal/secrets"
+	"vctp/internal/settings"
 	"vctp/internal/vcenter"
 	"vctp/server/handler"
 	"vctp/server/middleware"
 )
 
-func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver string, goVersion string, creds *vcenter.VcenterLogin, secret *secrets.Secrets) http.Handler {
+func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver string, goVersion string, creds *vcenter.VcenterLogin, secret *secrets.Secrets, settings *settings.Settings) http.Handler {
 	h := &handler.Handler{
 		Logger:    logger,
 		Database:  database,
@@ -21,6 +22,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
 		GoVersion: goVersion,
 		VcCreds:   creds,
 		Secret:    secret,
+		Settings:  settings,
 	}
 
 	mux := http.NewServeMux()
@@ -34,9 +36,13 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
 	mux.HandleFunc("/api/import/vm", h.VmImport)
 	// Use this when we need to manually remove a VM from the database to clean up
 	mux.HandleFunc("/api/inventory/vm/delete", h.VmCleanup)
+
+	// add missing data to VMs
+	mux.HandleFunc("/api/inventory/vm/update", h.VmUpdateDetails)
+
 	// temporary endpoint
 	//mux.HandleFunc("/api/cleanup/updates", h.UpdateCleanup)
-	mux.HandleFunc("/api/cleanup/vcenter", h.VcCleanup)
+	//mux.HandleFunc("/api/cleanup/vcenter", h.VcCleanup)
 
 	mux.HandleFunc("/api/report/inventory", h.InventoryReportDownload)
 	mux.HandleFunc("/api/report/updates", h.UpdateReportDownload)