From ea63ffa178716220bcca1fc71f5dad7c190d1be6 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Mon, 30 Sep 2024 12:01:39 +1000 Subject: [PATCH] various improvements --- .../20240915232747_extend_inventory.sql | 4 +- db/migrations/20240930012450_add_uuid.sql | 9 +++ db/queries/models.go | 1 + db/queries/query.sql | 8 ++- db/queries/query.sql.go | 67 ++++++++++++++++--- internal/settings/settings.go | 36 +++++++--- internal/tasks/monitorVcenter.go | 22 +++++- internal/tasks/processEvents.go | 5 ++ internal/tasks/tasks.go | 2 +- internal/utils/utils.go | 13 ++++ main.go | 10 +-- 11 files changed, 146 insertions(+), 31 deletions(-) create mode 100644 db/migrations/20240930012450_add_uuid.sql diff --git a/db/migrations/20240915232747_extend_inventory.sql b/db/migrations/20240915232747_extend_inventory.sql index 2ed89a9..67dc9d7 100644 --- a/db/migrations/20240915232747_extend_inventory.sql +++ b/db/migrations/20240915232747_extend_inventory.sql @@ -6,6 +6,6 @@ ALTER TABLE "Inventory" ADD COLUMN PowerState INTEGER; -- +goose Down -- +goose StatementBegin -ALTER TABLE "Updates" DROP COLUMN PowerState; -ALTER TABLE "Updates" DROP COLUMN IsTemplate; +ALTER TABLE "Inventory" DROP COLUMN PowerState; +ALTER TABLE "Inventory" DROP COLUMN IsTemplate; -- +goose StatementEnd diff --git a/db/migrations/20240930012450_add_uuid.sql b/db/migrations/20240930012450_add_uuid.sql new file mode 100644 index 0000000..606219e --- /dev/null +++ b/db/migrations/20240930012450_add_uuid.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE "Inventory" ADD COLUMN VmUuid TEXT; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE "Inventory" DROP COLUMN VmUuid; +-- +goose StatementEnd diff --git a/db/queries/models.go b/db/queries/models.go index 0cfcf3d..02e2851 100644 --- a/db/queries/models.go +++ b/db/queries/models.go @@ -46,6 +46,7 @@ type Inventory struct { IsTemplate interface{} PoweredOn interface{} SrmPlaceholder interface{} + VmUuid sql.NullString } type Updates struct { diff --git a/db/queries/query.sql b/db/queries/query.sql index 601cc47..d99a8fc 100644 --- a/db/queries/query.sql +++ b/db/queries/query.sql @@ -18,15 +18,19 @@ WHERE "Vcenter" = ?; SELECT * FROM "Inventory" WHERE "VmId" = sqlc.arg('vmId') AND "Datacenter" = sqlc.arg('datacenterName'); +-- name: GetInventoryVmUuid :one +SELECT * FROM "Inventory" +WHERE "VmUuid" = sqlc.arg('vmUuid') AND "Datacenter" = sqlc.arg('datacenterName'); + -- name: GetInventoryEventId :one SELECT * FROM "Inventory" WHERE "CloudId" = ? LIMIT 1; -- name: CreateInventory :one INSERT INTO "Inventory" ( - "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" + "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" ) VALUES( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) RETURNING *; diff --git a/db/queries/query.sql.go b/db/queries/query.sql.go index 53270ca..52c427d 100644 --- a/db/queries/query.sql.go +++ b/db/queries/query.sql.go @@ -90,17 +90,18 @@ func (q *Queries) CreateEvent(ctx context.Context, arg CreateEventParams) (Event const createInventory = `-- name: CreateInventory :one INSERT INTO "Inventory" ( - "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" + "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" ) VALUES( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` type CreateInventoryParams struct { Name string Vcenter string VmId sql.NullString + VmUuid sql.NullString EventKey sql.NullString CloudId sql.NullString CreationTime sql.NullInt64 @@ -122,6 +123,7 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams arg.Name, arg.Vcenter, arg.VmId, + arg.VmUuid, arg.EventKey, arg.CloudId, arg.CreationTime, @@ -158,6 +160,7 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ) return i, err } @@ -215,7 +218,7 @@ func (q *Queries) CreateUpdate(ctx context.Context, arg CreateUpdateParams) (Upd } const getInventoryByName = `-- name: GetInventoryByName :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" WHERE "Name" = ? ` @@ -248,6 +251,7 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ); err != nil { return nil, err } @@ -263,7 +267,7 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent } const getInventoryByVcenter = `-- name: GetInventoryByVcenter :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" WHERE "Vcenter" = ? ` @@ -296,6 +300,7 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([] &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ); err != nil { return nil, err } @@ -311,7 +316,7 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([] } const getInventoryEventId = `-- name: GetInventoryEventId :one -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" WHERE "CloudId" = ? LIMIT 1 ` @@ -338,12 +343,13 @@ func (q *Queries) GetInventoryEventId(ctx context.Context, cloudid sql.NullStrin &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ) return i, err } const getInventoryVmId = `-- name: GetInventoryVmId :one -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" WHERE "VmId" = ?1 AND "Datacenter" = ?2 ` @@ -375,12 +381,51 @@ func (q *Queries) GetInventoryVmId(ctx context.Context, arg GetInventoryVmIdPara &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, + ) + return i, err +} + +const getInventoryVmUuid = `-- name: GetInventoryVmUuid :one +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" +WHERE "VmUuid" = ?1 AND "Datacenter" = ?2 +` + +type GetInventoryVmUuidParams struct { + VmUuid sql.NullString + DatacenterName sql.NullString +} + +func (q *Queries) GetInventoryVmUuid(ctx context.Context, arg GetInventoryVmUuidParams) (Inventory, error) { + row := q.db.QueryRowContext(ctx, getInventoryVmUuid, arg.VmUuid, arg.DatacenterName) + var i Inventory + err := row.Scan( + &i.Iid, + &i.Name, + &i.Vcenter, + &i.VmId, + &i.EventKey, + &i.CloudId, + &i.CreationTime, + &i.DeletionTime, + &i.ResourcePool, + &i.VmType, + &i.Datacenter, + &i.Cluster, + &i.Folder, + &i.ProvisionedDisk, + &i.InitialVcpus, + &i.InitialRam, + &i.IsTemplate, + &i.PoweredOn, + &i.SrmPlaceholder, + &i.VmUuid, ) return i, err } const getReportInventory = `-- name: GetReportInventory :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" ORDER BY "CreationTime" ` @@ -413,6 +458,7 @@ func (q *Queries) GetReportInventory(ctx context.Context) ([]Inventory, error) { &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ); err != nil { return nil, err } @@ -470,7 +516,7 @@ func (q *Queries) GetReportUpdates(ctx context.Context) ([]Updates, error) { const inventoryCleanup = `-- name: InventoryCleanup :exec DELETE FROM "Inventory" WHERE "VmId" = ?1 AND "Datacenter" = ?2 -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` type InventoryCleanupParams struct { @@ -545,7 +591,7 @@ func (q *Queries) ListEvents(ctx context.Context) ([]Events, error) { } const listInventory = `-- name: ListInventory :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory" +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory" ORDER BY "Name" ` @@ -578,6 +624,7 @@ func (q *Queries) ListInventory(ctx context.Context) ([]Inventory, error) { &i.IsTemplate, &i.PoweredOn, &i.SrmPlaceholder, + &i.VmUuid, ); err != nil { return nil, err } diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 655f3e9..abbafd1 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -1,6 +1,8 @@ package settings import ( + "errors" + "fmt" "log/slog" "os" "vctp/internal/utils" @@ -8,6 +10,12 @@ import ( "gopkg.in/yaml.v2" ) +type Settings struct { + SettingsPath string + Logger *slog.Logger + Values *SettingsYML +} + // SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties type SettingsYML struct { Settings struct { @@ -18,23 +26,30 @@ type SettingsYML struct { } `yaml:"settings"` } -func ReadYMLSettings(logger *slog.Logger, settingsPath string) (SettingsYML, error) { +func New(logger *slog.Logger, settingsPath string) *Settings { + return &Settings{ + SettingsPath: utils.GetFilePath(settingsPath), + Logger: logger, + } +} + +func (s *Settings) ReadYMLSettings() error { // Create config structure var settings SettingsYML // Check for empty filename - if len(settingsPath) == 0 { - return settings, nil + if len(s.SettingsPath) == 0 { + return errors.New("settings file path not specified") } - path := utils.GetFilePath(settingsPath) + //path := utils.GetFilePath(settingsPath) // Open config file - file, err := os.Open(path) + file, err := os.Open(s.SettingsPath) if err != nil { - return settings, err + return fmt.Errorf("unable to open settings file : '%s'", err) } - logger.Debug("Opened settings yaml file", "file_path", path) + s.Logger.Debug("Opened settings yaml file", "file_path", s.SettingsPath) defer file.Close() // Init new YAML decode @@ -42,8 +57,11 @@ func ReadYMLSettings(logger *slog.Logger, settingsPath string) (SettingsYML, err // Start YAML decoding from file if err := d.Decode(&settings); err != nil { - return settings, err + return fmt.Errorf("unable to decode settings file : '%s'", err) } - return settings, nil + s.Logger.Debug("Updating settings", "settings", settings) + s.Values = &settings + + return nil } diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index e3ac35c..249ed05 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -6,7 +6,10 @@ import ( "errors" "fmt" "log/slog" + "strings" + "time" "vctp/db/queries" + "vctp/internal/utils" "vctp/internal/vcenter" "github.com/vmware/govmomi/vim25/mo" @@ -17,9 +20,10 @@ import ( func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) error { var matchFound bool - // TODO - reload settings in case vcenter list has changed + // reload settings in case vcenter list has changed + c.Settings.ReadYMLSettings() - for _, url := range c.Settings.Settings.VcenterAddresses { + for _, url := range c.Settings.Values.Settings.VcenterAddresses { c.Logger.Debug("connecting to vcenter", "url", url) vc := vcenter.New(c.Logger, c.VcCreds) vc.Login(url) @@ -43,9 +47,17 @@ func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) erro // Iterate VMs from vcenter and see if they were in the database for _, vm := range vms { matchFound = false + + // Skip any vCLS VMs + if strings.HasPrefix(vm.Name(), "vCLS-") { + c.Logger.Debug("Skipping internal VM", "vm_name", vm.Name()) + continue + } + + // TODO - should we compare the UUID as well? for _, dbvm := range results { if dbvm.VmId.String == vm.Reference().Value { - c.Logger.Debug("Found match for VM", "name", dbvm.Name, "id", dbvm.VmId.String) + c.Logger.Debug("Found match for VM", "vm_name", dbvm.Name, "id", dbvm.VmId.String) matchFound = true break } @@ -61,6 +73,9 @@ func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) erro // retrieve VM properties and insert into inventory c.AddVmToInventory(vmObj, vc, ctx) + + // add sleep to slow down mass VM additions + utils.SleepWithContext(ctx, (10 * time.Millisecond)) } } c.Logger.Debug("Finished checking vcenter", "url", url) @@ -190,6 +205,7 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce ProvisionedDisk: sql.NullFloat64{Float64: totalDiskGB, Valid: totalDiskGB > 0}, Folder: sql.NullString{String: folderPath, Valid: folderPath != ""}, ResourcePool: sql.NullString{String: rpName, Valid: rpName != ""}, + VmUuid: sql.NullString{String: vmObject.Config.Uuid, Valid: vmObject.Config.Uuid != ""}, SrmPlaceholder: srmPlaceholder, IsTemplate: isTemplate, PoweredOn: poweredOn, diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index 1def465..23514f7 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -23,6 +23,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { poweredOn string folderPath string rpName string + vmUuid string ) dateCmp := time.Now().AddDate(0, 0, -1).Unix() @@ -60,6 +61,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { totalDiskGB = 0 isTemplate = "FALSE" folderPath = "" + vmUuid = "" } else { c.Logger.Debug("found VM") srmPlaceholder = "FALSE" // Default assumption @@ -69,6 +71,8 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { if vmObject.Config != nil { numRam = vmObject.Config.Hardware.MemoryMB numVcpus = vmObject.Config.Hardware.NumCPU + vmUuid = vmObject.Config.Uuid + var totalDiskBytes int64 // Calculate the total disk allocated in GB @@ -148,6 +152,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { ProvisionedDisk: sql.NullFloat64{Float64: totalDiskGB, Valid: totalDiskGB > 0}, Folder: sql.NullString{String: folderPath, Valid: folderPath != ""}, ResourcePool: sql.NullString{String: rpName, Valid: rpName != ""}, + VmUuid: sql.NullString{String: vmUuid, Valid: vmUuid != ""}, SrmPlaceholder: srmPlaceholder, IsTemplate: isTemplate, PoweredOn: poweredOn, diff --git a/internal/tasks/tasks.go b/internal/tasks/tasks.go index 39ab30c..4aa3c9d 100644 --- a/internal/tasks/tasks.go +++ b/internal/tasks/tasks.go @@ -11,6 +11,6 @@ import ( type CronTask struct { Logger *slog.Logger Database db.Database - Settings settings.SettingsYML + Settings *settings.Settings VcCreds *vcenter.VcenterLogin } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 5f7057a..85c09a6 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -1,11 +1,13 @@ package utils import ( + "context" "log" "log/slog" "net" "os" "path/filepath" + "time" ) const rsaBits = 4096 @@ -53,3 +55,14 @@ func FileExists(filename string) bool { } return !info.IsDir() } + +func SleepWithContext(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + } +} diff --git a/main.go b/main.go index 7904587..301208e 100644 --- a/main.go +++ b/main.go @@ -66,7 +66,9 @@ func main() { } // TODO - how to pass this to the other packages that will need this info? - s, err := settings.ReadYMLSettings(logger, settingsFile) + s := settings.New(logger, settingsFile) + err = s.ReadYMLSettings() + //s, err := settings.ReadYMLSettings(logger, settingsFile) if err != nil { logger.Error("failed to open yaml settings file", "error", err, "filename", settingsFile) //os.Exit(1) @@ -165,11 +167,11 @@ func main() { if cronInventoryFrequencyString != "" { cronInvFrequency, err = time.ParseDuration(cronInventoryFrequencyString) if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 3600s", "value", cronInventoryFrequencyString, "error", err) - cronInvFrequency = time.Second * 3600 + slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 7200", "value", cronInventoryFrequencyString, "error", err) + cronInvFrequency = time.Second * 7200 } } else { - cronInvFrequency = time.Second * 3600 + cronInvFrequency = time.Second * 7200 } logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency)