diff --git a/db/migrations/20240913043145_extend_events.sql b/db/migrations/20240913043145_extend_events.sql index ee0173e..dbdbb9d 100644 --- a/db/migrations/20240913043145_extend_events.sql +++ b/db/migrations/20240913043145_extend_events.sql @@ -10,8 +10,8 @@ ALTER TABLE "Events" ADD COLUMN VmName TEXT; -- +goose Down -- +goose StatementBegin ALTER TABLE "Events" DROP COLUMN VmName; -ALTER TABLE "Updates" DROP COLUMN ComputeResourceId; -ALTER TABLE "Updates" DROP COLUMN DatacenterId; +ALTER TABLE "Events" DROP COLUMN ComputeResourceId; +ALTER TABLE "Events" DROP COLUMN DatacenterId; ALTER TABLE "Events" RENAME COLUMN ComputeResourceName to ComputeResource; ALTER TABLE "Events" RENAME COLUMN DatacenterName to Datacenter; -- +goose StatementEnd diff --git a/db/migrations/20240930031506_add_table.sql b/db/migrations/20240930031506_add_table.sql new file mode 100644 index 0000000..76989cc --- /dev/null +++ b/db/migrations/20240930031506_add_table.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS "InventoryHistory" ( + "Hid" INTEGER PRIMARY KEY AUTOINCREMENT, + "InventoryId" INTEGER, + "ReportDate" INTEGER, + "UpdateTime" INTEGER, + "PreviousVcpus" INTEGER, + "PreviousRam" INTEGER, + "PreviousResourcePool" TEXT, + "PreviousProvisionedDisk" REAL +) +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE "InventoryHistory"; +-- +goose StatementEnd diff --git a/db/queries/models.go b/db/queries/models.go index 02e2851..e6056bd 100644 --- a/db/queries/models.go +++ b/db/queries/models.go @@ -49,6 +49,17 @@ type Inventory struct { VmUuid sql.NullString } +type InventoryHistory struct { + Hid int64 + InventoryId sql.NullInt64 + ReportDate sql.NullInt64 + UpdateTime sql.NullInt64 + PreviousVcpus sql.NullInt64 + PreviousRam sql.NullInt64 + PreviousResourcePool sql.NullString + PreviousProvisionedDisk sql.NullFloat64 +} + type Updates struct { Uid int64 InventoryId sql.NullInt64 diff --git a/db/queries/query.sql b/db/queries/query.sql index 08b633b..21b0b87 100644 --- a/db/queries/query.sql +++ b/db/queries/query.sql @@ -38,6 +38,11 @@ INSERT INTO "Inventory" ( ) RETURNING *; +-- name: InventoryUpdate :exec +UPDATE "Inventory" +SET "VmUuid" = sqlc.arg('uuid'), "SrmPlaceholder" = sqlc.arg('srmPlaceholder') +WHERE "Iid" = sqlc.arg('iid'); + -- name: InventoryMarkDeleted :exec UPDATE "Inventory" SET "DeletionTime" = sqlc.arg('deletionTime') @@ -91,4 +96,12 @@ ORDER BY "EventTime"; -- name: UpdateEventsProcessed :exec UPDATE "Events" SET "Processed" = 1 -WHERE "Eid" = sqlc.arg('eid'); \ No newline at end of file +WHERE "Eid" = sqlc.arg('eid'); + +-- name: CreateInventoryHistory :one +INSERT INTO "InventoryHistory" ( + "InventoryId", "ReportDate", "UpdateTime", "PreviousVcpus", "PreviousRam", "PreviousResourcePool", "PreviousProvisionedDisk" +) VALUES( + ?, ?, ?, ?, ?, ?, ? +) +RETURNING *; \ No newline at end of file diff --git a/db/queries/query.sql.go b/db/queries/query.sql.go index dc7e505..e59671e 100644 --- a/db/queries/query.sql.go +++ b/db/queries/query.sql.go @@ -165,6 +165,49 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams return i, err } +const createInventoryHistory = `-- name: CreateInventoryHistory :one +INSERT INTO "InventoryHistory" ( + "InventoryId", "ReportDate", "UpdateTime", "PreviousVcpus", "PreviousRam", "PreviousResourcePool", "PreviousProvisionedDisk" +) VALUES( + ?, ?, ?, ?, ?, ?, ? +) +RETURNING Hid, InventoryId, ReportDate, UpdateTime, PreviousVcpus, PreviousRam, PreviousResourcePool, PreviousProvisionedDisk +` + +type CreateInventoryHistoryParams struct { + InventoryId sql.NullInt64 + ReportDate sql.NullInt64 + UpdateTime sql.NullInt64 + PreviousVcpus sql.NullInt64 + PreviousRam sql.NullInt64 + PreviousResourcePool sql.NullString + PreviousProvisionedDisk sql.NullFloat64 +} + +func (q *Queries) CreateInventoryHistory(ctx context.Context, arg CreateInventoryHistoryParams) (InventoryHistory, error) { + row := q.db.QueryRowContext(ctx, createInventoryHistory, + arg.InventoryId, + arg.ReportDate, + arg.UpdateTime, + arg.PreviousVcpus, + arg.PreviousRam, + arg.PreviousResourcePool, + arg.PreviousProvisionedDisk, + ) + var i InventoryHistory + err := row.Scan( + &i.Hid, + &i.InventoryId, + &i.ReportDate, + &i.UpdateTime, + &i.PreviousVcpus, + &i.PreviousRam, + &i.PreviousResourcePool, + &i.PreviousProvisionedDisk, + ) + return i, err +} + const createUpdate = `-- name: CreateUpdate :one INSERT INTO "Updates" ( "InventoryId", "EventKey", "EventId", "UpdateTime", "UpdateType", "NewVcpus", "NewRam", "NewResourcePool", "NewProvisionedDisk", "UserName" @@ -606,6 +649,23 @@ func (q *Queries) InventoryMarkDeleted(ctx context.Context, arg InventoryMarkDel return err } +const inventoryUpdate = `-- name: InventoryUpdate :exec +UPDATE "Inventory" +SET "VmUuid" = ?1, "SrmPlaceholder" = ?2 +WHERE "Iid" = ?3 +` + +type InventoryUpdateParams struct { + Uuid sql.NullString + SrmPlaceholder interface{} + Iid int64 +} + +func (q *Queries) InventoryUpdate(ctx context.Context, arg InventoryUpdateParams) error { + _, err := q.db.ExecContext(ctx, inventoryUpdate, arg.Uuid, arg.SrmPlaceholder, arg.Iid) + return err +} + const listEvents = `-- name: ListEvents :many SELECT Eid, CloudId, Source, EventTime, ChainId, VmId, EventKey, DatacenterName, ComputeResourceName, UserName, Processed, DatacenterId, ComputeResourceId, VmName, EventType FROM "Events" ORDER BY "EventTime" diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index 30c4ee8..01945e8 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -114,16 +114,16 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce c.Logger.Debug("found VM") - if vmObject.Name == "DBRaaS_testVMTemplate" { - c.Logger.Debug("Found problematic VM") - //prettyPrint(vmObject) - } - - srmPlaceholder = "FALSE" // Default assumption - //prettyPrint(vmObject) + /* + if vmObject.Name == "DBRaaS_testVMTemplate" { + c.Logger.Debug("Found problematic VM") + //prettyPrint(vmObject) + } + */ // calculate VM properties we want to store if vmObject.Config != nil { + // Skip any template VMs if vmObject.Config.Template { c.Logger.Debug("Not adding templates to inventory") return nil @@ -133,6 +133,7 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce numRam = vmObject.Config.Hardware.MemoryMB numVcpus = vmObject.Config.Hardware.NumCPU + srmPlaceholder = "FALSE" // Default assumption // Calculate creation date if vmObject.Config.CreateDate.IsZero() { @@ -156,16 +157,19 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce } totalDiskBytes += disk.CapacityInBytes - //totalDiskGB += float64(disk.CapacityInBytes / 1024 / 1024 / 1024) // Convert from bytes to GB } } totalDiskGB = float64(totalDiskBytes / 1024 / 1024 / 1024) c.Logger.Debug("Converted total disk size", "bytes", totalDiskBytes, "GB", totalDiskGB) // Determine if the VM is a normal VM or an SRM placeholder - if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.Type == "com.vmware.vcDr" { - c.Logger.Debug("VM ManagedBy indicates managed by SRM") - srmPlaceholder = "TRUE" + if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" { + if vmObject.Config.ManagedBy.Type == "placeholderVm" { + c.Logger.Debug("VM is a placeholder") + srmPlaceholder = "TRUE" + } else { + c.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObject.Config.ManagedBy) + } } // Retrieve the full folder path of the VM @@ -179,10 +183,10 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce foundVmConfig = true } else { - c.Logger.Error("Empty VM config") + c.Logger.Warn("Empty VM config") } - c.Logger.Debug("VM has runtime data", "power_state", vmObject.Runtime.PowerState) + //c.Logger.Debug("VM has runtime data", "power_state", vmObject.Runtime.PowerState) if vmObject.Runtime.PowerState == "poweredOff" { poweredOn = "FALSE" } else { @@ -241,7 +245,6 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce } return nil - } // prettyPrint comes from https://gist.github.com/sfate/9d45f6c5405dc4c9bf63bf95fe6d1a7c diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index 23514f7..219cf9e 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -94,9 +94,13 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { c.Logger.Debug("Converted total disk size", "bytes", totalDiskBytes, "GB", totalDiskGB) // Determine if the VM is a normal VM or an SRM placeholder - if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.Type == "com.vmware.vcDr" { - c.Logger.Debug("VM ManagedBy indicates managed by SRM") - srmPlaceholder = "TRUE" + if vmObject.Config.ManagedBy != nil && vmObject.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" { + if vmObject.Config.ManagedBy.Type == "placeholderVm" { + c.Logger.Debug("VM is a placeholder") + srmPlaceholder = "TRUE" + } else { + c.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObject.Config.ManagedBy) + } } if vmObject.Config.Template { diff --git a/main.go b/main.go index 301208e..c9fb670 100644 --- a/main.go +++ b/main.go @@ -209,7 +209,7 @@ func main() { c.Start() // Start server - r := router.New(logger, database, buildTime, sha1ver, runtime.Version(), &creds, a) + r := router.New(logger, database, buildTime, sha1ver, runtime.Version(), &creds, a, s) svr := server.New( logger, c, diff --git a/server/handler/handler.go b/server/handler/handler.go index d75cf32..ac31025 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -6,6 +6,7 @@ import ( "net/http" "vctp/db" "vctp/internal/secrets" + "vctp/internal/settings" "vctp/internal/vcenter" "github.com/a-h/templ" @@ -20,6 +21,7 @@ type Handler struct { GoVersion string VcCreds *vcenter.VcenterLogin Secret *secrets.Secrets + Settings *settings.Settings } func (h *Handler) html(ctx context.Context, w http.ResponseWriter, status int, t templ.Component) { diff --git a/server/handler/vmUpdateDetails.go b/server/handler/vmUpdateDetails.go new file mode 100644 index 0000000..93bdb51 --- /dev/null +++ b/server/handler/vmUpdateDetails.go @@ -0,0 +1,114 @@ +package handler + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "vctp/db/queries" + "vctp/internal/vcenter" +) + +// VmUpdate receives the CloudEvent for a VM modification or move +func (h *Handler) VmUpdateDetails(w http.ResponseWriter, r *http.Request) { + var matchFound bool + var inventoryId int64 + var srmPlaceholder string + var vmUuid string + var dbUuid string + + ctx := context.Background() + + // reload settings in case vcenter list has changed + h.Settings.ReadYMLSettings() + + for _, url := range h.Settings.Values.Settings.VcenterAddresses { + h.Logger.Debug("connecting to vcenter", "url", url) + vc := vcenter.New(h.Logger, h.VcCreds) + vc.Login(url) + + // Get list of VMs from vcenter + vms, err := vc.GetAllVmReferences() + + // Get list of VMs from inventory table + h.Logger.Debug("Querying inventory table") + results, err := h.Database.Queries().GetInventoryByVcenter(ctx, url) + if err != nil { + h.Logger.Error("Unable to query inventory table", "error", err) + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to query inventory table %s\n", err) + } + + if len(results) == 0 { + h.Logger.Error("Empty inventory results") + } + + // Iterate VMs from vcenter and see if they were in the database + for _, vm := range vms { + matchFound = false + inventoryId = 0 + srmPlaceholder = "FALSE" // Default assumption + vmUuid = "" + + for _, dbvm := range results { + if dbvm.VmId.String == vm.Reference().Value { + h.Logger.Debug("Found VM in database", "vm_name", dbvm.Name, "id", dbvm.VmId.String) + matchFound = true + inventoryId = dbvm.Iid + dbUuid = dbvm.VmUuid.String + break + } + } + + if matchFound { + //h.Logger.Debug("Need to update VM in inventory table", "MoRef", vm.Reference()) + vmObj, err := vc.ConvertObjToMoVM(vm) + if err != nil { + h.Logger.Error("Received error getting vm managedobject", "error", err) + continue + } + + if vmObj.Config != nil { + vmUuid = vmObj.Config.Uuid + + // Determine if the VM is a normal VM or an SRM placeholder + if vmObj.Config.ManagedBy != nil && vmObj.Config.ManagedBy.ExtensionKey == "com.vmware.vcDr" { + if vmObj.Config.ManagedBy.Type == "placeholderVm" { + h.Logger.Debug("VM is a placeholder") + srmPlaceholder = "TRUE" + } else { + h.Logger.Debug("VM is managed by SRM but not a placeholder", "details", vmObj.Config.ManagedBy) + } + } + + if srmPlaceholder == "TRUE" || vmUuid != dbUuid { + h.Logger.Debug("Need to update vm", "name", vmObj.Name, "srm_placeholder", srmPlaceholder, "uuid", vmUuid) + params := queries.InventoryUpdateParams{ + Iid: inventoryId, + SrmPlaceholder: srmPlaceholder, + Uuid: sql.NullString{String: vmUuid, Valid: vmUuid != ""}, + } + + h.Logger.Debug("database params", "params", params) + err := h.Database.Queries().InventoryUpdate(context.Background(), params) + + if err != nil { + h.Logger.Error("Error received updating inventory for VM", "name", vmObj.Name, "error", err) + } + + } + + } else { + h.Logger.Warn("VM no longer present in vcenter or missing config values", "MoRef", vm.Reference()) + } + + } + + } + } + + h.Logger.Debug("Processed vm update successfully") + w.WriteHeader(http.StatusOK) + // TODO - return some JSON + fmt.Fprintf(w, "Processed vm update successfully") +} diff --git a/server/router/router.go b/server/router/router.go index 58b7f77..9e5d27f 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -7,12 +7,13 @@ import ( "vctp/db" "vctp/dist" "vctp/internal/secrets" + "vctp/internal/settings" "vctp/internal/vcenter" "vctp/server/handler" "vctp/server/middleware" ) -func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver string, goVersion string, creds *vcenter.VcenterLogin, secret *secrets.Secrets) http.Handler { +func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver string, goVersion string, creds *vcenter.VcenterLogin, secret *secrets.Secrets, settings *settings.Settings) http.Handler { h := &handler.Handler{ Logger: logger, Database: database, @@ -21,6 +22,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st GoVersion: goVersion, VcCreds: creds, Secret: secret, + Settings: settings, } mux := http.NewServeMux() @@ -34,9 +36,13 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st mux.HandleFunc("/api/import/vm", h.VmImport) // Use this when we need to manually remove a VM from the database to clean up mux.HandleFunc("/api/inventory/vm/delete", h.VmCleanup) + + // add missing data to VMs + mux.HandleFunc("/api/inventory/vm/update", h.VmUpdateDetails) + // temporary endpoint //mux.HandleFunc("/api/cleanup/updates", h.UpdateCleanup) - mux.HandleFunc("/api/cleanup/vcenter", h.VcCleanup) + //mux.HandleFunc("/api/cleanup/vcenter", h.VcCleanup) mux.HandleFunc("/api/report/inventory", h.InventoryReportDownload) mux.HandleFunc("/api/report/updates", h.UpdateReportDownload)