various improvements

db/migrations/20240915232747_extend_inventory.sql

@@ -6,6 +6,6 @@ ALTER TABLE "Inventory" ADD COLUMN PowerState INTEGER;
 
 -- +goose Down
 -- +goose StatementBegin
-ALTER TABLE "Updates" DROP COLUMN PowerState;
-ALTER TABLE "Updates" DROP COLUMN IsTemplate;
+ALTER TABLE "Inventory" DROP COLUMN PowerState;
+ALTER TABLE "Inventory" DROP COLUMN IsTemplate;
 -- +goose StatementEnd

db/migrations/20240930012450_add_uuid.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Inventory" ADD COLUMN VmUuid TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Inventory" DROP COLUMN VmUuid;
+-- +goose StatementEnd

db/queries/models.go

@@ -46,6 +46,7 @@ type Inventory struct {
 	IsTemplate      interface{}
 	PoweredOn       interface{}
 	SrmPlaceholder  interface{}
+	VmUuid          sql.NullString
 }
 
 type Updates struct {

db/queries/query.sql

@@ -18,15 +18,19 @@ WHERE "Vcenter" = ?;
 SELECT * FROM "Inventory"
 WHERE "VmId" = sqlc.arg('vmId')  AND "Datacenter" = sqlc.arg('datacenterName');
 
+-- name: GetInventoryVmUuid :one
+SELECT * FROM "Inventory"
+WHERE "VmUuid" = sqlc.arg('vmUuid')  AND "Datacenter" = sqlc.arg('datacenterName');
+
 -- name: GetInventoryEventId :one
 SELECT * FROM "Inventory"
 WHERE "CloudId" = ? LIMIT 1;
 
 -- name: CreateInventory :one
 INSERT INTO "Inventory" (
-    "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn"
+    "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn"
 ) VALUES(
-    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
 )
 RETURNING *;
 

db/queries/query.sql.go

@@ -90,17 +90,18 @@ func (q *Queries) CreateEvent(ctx context.Context, arg CreateEventParams) (Event
 
 const createInventory = `-- name: CreateInventory :one
 INSERT INTO "Inventory" (
-    "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn"
+    "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn"
 ) VALUES(
-    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
 )
-RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder
+RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid
 `
 
 type CreateInventoryParams struct {
 	Name            string
 	Vcenter         string
 	VmId            sql.NullString
+	VmUuid          sql.NullString
 	EventKey        sql.NullString
 	CloudId         sql.NullString
 	CreationTime    sql.NullInt64
@@ -122,6 +123,7 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams
 		arg.Name,
 		arg.Vcenter,
 		arg.VmId,
+		arg.VmUuid,
 		arg.EventKey,
 		arg.CloudId,
 		arg.CreationTime,
@@ -158,6 +160,7 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams
 		&i.IsTemplate,
 		&i.PoweredOn,
 		&i.SrmPlaceholder,
+		&i.VmUuid,
 	)
 	return i, err
 }
@@ -215,7 +218,7 @@ func (q *Queries) CreateUpdate(ctx context.Context, arg CreateUpdateParams) (Upd
 }
 
 const getInventoryByName = `-- name: GetInventoryByName :many
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 WHERE "Name" = ?
 `
 
@@ -248,6 +251,7 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent
 			&i.IsTemplate,
 			&i.PoweredOn,
 			&i.SrmPlaceholder,
+			&i.VmUuid,
 		); err != nil {
 			return nil, err
 		}
@@ -263,7 +267,7 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent
 }
 
 const getInventoryByVcenter = `-- name: GetInventoryByVcenter :many
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 WHERE "Vcenter" = ?
 `
 
@@ -296,6 +300,7 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([]
 			&i.IsTemplate,
 			&i.PoweredOn,
 			&i.SrmPlaceholder,
+			&i.VmUuid,
 		); err != nil {
 			return nil, err
 		}
@@ -311,7 +316,7 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([]
 }
 
 const getInventoryEventId = `-- name: GetInventoryEventId :one
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 WHERE "CloudId" = ? LIMIT 1
 `
 
@@ -338,12 +343,13 @@ func (q *Queries) GetInventoryEventId(ctx context.Context, cloudid sql.NullStrin
 		&i.IsTemplate,
 		&i.PoweredOn,
 		&i.SrmPlaceholder,
+		&i.VmUuid,
 	)
 	return i, err
 }
 
 const getInventoryVmId = `-- name: GetInventoryVmId :one
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 WHERE "VmId" = ?1  AND "Datacenter" = ?2
 `
 
@@ -375,12 +381,51 @@ func (q *Queries) GetInventoryVmId(ctx context.Context, arg GetInventoryVmIdPara
 		&i.IsTemplate,
 		&i.PoweredOn,
 		&i.SrmPlaceholder,
+		&i.VmUuid,
+	)
+	return i, err
+}
+
+const getInventoryVmUuid = `-- name: GetInventoryVmUuid :one
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
+WHERE "VmUuid" = ?1  AND "Datacenter" = ?2
+`
+
+type GetInventoryVmUuidParams struct {
+	VmUuid         sql.NullString
+	DatacenterName sql.NullString
+}
+
+func (q *Queries) GetInventoryVmUuid(ctx context.Context, arg GetInventoryVmUuidParams) (Inventory, error) {
+	row := q.db.QueryRowContext(ctx, getInventoryVmUuid, arg.VmUuid, arg.DatacenterName)
+	var i Inventory
+	err := row.Scan(
+		&i.Iid,
+		&i.Name,
+		&i.Vcenter,
+		&i.VmId,
+		&i.EventKey,
+		&i.CloudId,
+		&i.CreationTime,
+		&i.DeletionTime,
+		&i.ResourcePool,
+		&i.VmType,
+		&i.Datacenter,
+		&i.Cluster,
+		&i.Folder,
+		&i.ProvisionedDisk,
+		&i.InitialVcpus,
+		&i.InitialRam,
+		&i.IsTemplate,
+		&i.PoweredOn,
+		&i.SrmPlaceholder,
+		&i.VmUuid,
 	)
 	return i, err
 }
 
 const getReportInventory = `-- name: GetReportInventory :many
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 ORDER BY "CreationTime"
 `
 
@@ -413,6 +458,7 @@ func (q *Queries) GetReportInventory(ctx context.Context) ([]Inventory, error) {
 			&i.IsTemplate,
 			&i.PoweredOn,
 			&i.SrmPlaceholder,
+			&i.VmUuid,
 		); err != nil {
 			return nil, err
 		}
@@ -470,7 +516,7 @@ func (q *Queries) GetReportUpdates(ctx context.Context) ([]Updates, error) {
 const inventoryCleanup = `-- name: InventoryCleanup :exec
 DELETE FROM "Inventory"
 WHERE "VmId" = ?1 AND "Datacenter" = ?2
-RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder
+RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid
 `
 
 type InventoryCleanupParams struct {
@@ -545,7 +591,7 @@ func (q *Queries) ListEvents(ctx context.Context) ([]Events, error) {
 }
 
 const listInventory = `-- name: ListInventory :many
-SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder FROM "Inventory"
+SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM "Inventory"
 ORDER BY "Name"
 `
 
@@ -578,6 +624,7 @@ func (q *Queries) ListInventory(ctx context.Context) ([]Inventory, error) {
 			&i.IsTemplate,
 			&i.PoweredOn,
 			&i.SrmPlaceholder,
+			&i.VmUuid,
 		); err != nil {
 			return nil, err
 		}

internal/settings/settings.go

@@ -1,6 +1,8 @@
 package settings
 
 import (
+	"errors"
+	"fmt"
 	"log/slog"
 	"os"
 	"vctp/internal/utils"
@@ -8,6 +10,12 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+type Settings struct {
+	SettingsPath string
+	Logger       *slog.Logger
+	Values       *SettingsYML
+}
+
 // SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties
 type SettingsYML struct {
 	Settings struct {
@@ -18,23 +26,30 @@ type SettingsYML struct {
 	} `yaml:"settings"`
 }
 
-func ReadYMLSettings(logger *slog.Logger, settingsPath string) (SettingsYML, error) {
+func New(logger *slog.Logger, settingsPath string) *Settings {
+	return &Settings{
+		SettingsPath: utils.GetFilePath(settingsPath),
+		Logger:       logger,
+	}
+}
+
+func (s *Settings) ReadYMLSettings() error {
 	// Create config structure
 	var settings SettingsYML
 
 	// Check for empty filename
-	if len(settingsPath) == 0 {
-		return settings, nil
+	if len(s.SettingsPath) == 0 {
+		return errors.New("settings file path not specified")
 	}
 
-	path := utils.GetFilePath(settingsPath)
+	//path := utils.GetFilePath(settingsPath)
 
 	// Open config file
-	file, err := os.Open(path)
+	file, err := os.Open(s.SettingsPath)
 	if err != nil {
-		return settings, err
+		return fmt.Errorf("unable to open settings file : '%s'", err)
 	}
-	logger.Debug("Opened settings yaml file", "file_path", path)
+	s.Logger.Debug("Opened settings yaml file", "file_path", s.SettingsPath)
 	defer file.Close()
 
 	// Init new YAML decode
@@ -42,8 +57,11 @@ func ReadYMLSettings(logger *slog.Logger, settingsPath string) (SettingsYML, err
 
 	// Start YAML decoding from file
 	if err := d.Decode(&settings); err != nil {
-		return settings, err
+		return fmt.Errorf("unable to decode settings file : '%s'", err)
 	}
 
-	return settings, nil
+	s.Logger.Debug("Updating settings", "settings", settings)
+	s.Values = &settings
+
+	return nil
 }

internal/tasks/monitorVcenter.go

@@ -6,7 +6,10 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"strings"
+	"time"
 	"vctp/db/queries"
+	"vctp/internal/utils"
 	"vctp/internal/vcenter"
 
 	"github.com/vmware/govmomi/vim25/mo"
@@ -17,9 +20,10 @@ import (
 func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) error {
 	var matchFound bool
 
-	// TODO - reload settings in case vcenter list has changed
+	// reload settings in case vcenter list has changed
+	c.Settings.ReadYMLSettings()
 
-	for _, url := range c.Settings.Settings.VcenterAddresses {
+	for _, url := range c.Settings.Values.Settings.VcenterAddresses {
 		c.Logger.Debug("connecting to vcenter", "url", url)
 		vc := vcenter.New(c.Logger, c.VcCreds)
 		vc.Login(url)
@@ -43,9 +47,17 @@ func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) erro
 		// Iterate VMs from vcenter and see if they were in the database
 		for _, vm := range vms {
 			matchFound = false
+
+			// Skip any vCLS VMs
+			if strings.HasPrefix(vm.Name(), "vCLS-") {
+				c.Logger.Debug("Skipping internal VM", "vm_name", vm.Name())
+				continue
+			}
+
+			// TODO - should we compare the UUID as well?
 			for _, dbvm := range results {
 				if dbvm.VmId.String == vm.Reference().Value {
-					c.Logger.Debug("Found match for VM", "name", dbvm.Name, "id", dbvm.VmId.String)
+					c.Logger.Debug("Found match for VM", "vm_name", dbvm.Name, "id", dbvm.VmId.String)
 					matchFound = true
 					break
 				}
@@ -61,6 +73,9 @@ func (c *CronTask) RunVcenterPoll(ctx context.Context, logger *slog.Logger) erro
 
 				// retrieve VM properties and insert into inventory
 				c.AddVmToInventory(vmObj, vc, ctx)
+
+				// add sleep to slow down mass VM additions
+				utils.SleepWithContext(ctx, (10 * time.Millisecond))
 			}
 		}
 		c.Logger.Debug("Finished checking vcenter", "url", url)
@@ -190,6 +205,7 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
 			ProvisionedDisk: sql.NullFloat64{Float64: totalDiskGB, Valid: totalDiskGB > 0},
 			Folder:          sql.NullString{String: folderPath, Valid: folderPath != ""},
 			ResourcePool:    sql.NullString{String: rpName, Valid: rpName != ""},
+			VmUuid:          sql.NullString{String: vmObject.Config.Uuid, Valid: vmObject.Config.Uuid != ""},
 			SrmPlaceholder:  srmPlaceholder,
 			IsTemplate:      isTemplate,
 			PoweredOn:       poweredOn,

internal/tasks/processEvents.go

@@ -23,6 +23,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
 		poweredOn      string
 		folderPath     string
 		rpName         string
+		vmUuid         string
 	)
 
 	dateCmp := time.Now().AddDate(0, 0, -1).Unix()
@@ -60,6 +61,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
 			totalDiskGB = 0
 			isTemplate = "FALSE"
 			folderPath = ""
+			vmUuid = ""
 		} else {
 			c.Logger.Debug("found VM")
 			srmPlaceholder = "FALSE" // Default assumption
@@ -69,6 +71,8 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
 			if vmObject.Config != nil {
 				numRam = vmObject.Config.Hardware.MemoryMB
 				numVcpus = vmObject.Config.Hardware.NumCPU
+				vmUuid = vmObject.Config.Uuid
+
 				var totalDiskBytes int64
 
 				// Calculate the total disk allocated in GB
@@ -148,6 +152,7 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
 				ProvisionedDisk: sql.NullFloat64{Float64: totalDiskGB, Valid: totalDiskGB > 0},
 				Folder:          sql.NullString{String: folderPath, Valid: folderPath != ""},
 				ResourcePool:    sql.NullString{String: rpName, Valid: rpName != ""},
+				VmUuid:          sql.NullString{String: vmUuid, Valid: vmUuid != ""},
 				SrmPlaceholder:  srmPlaceholder,
 				IsTemplate:      isTemplate,
 				PoweredOn:       poweredOn,

internal/tasks/tasks.go

@@ -11,6 +11,6 @@ import (
 type CronTask struct {
 	Logger   *slog.Logger
 	Database db.Database
-	Settings settings.SettingsYML
+	Settings *settings.Settings
 	VcCreds  *vcenter.VcenterLogin
 }

internal/utils/utils.go

@@ -1,11 +1,13 @@
 package utils
 
 import (
+	"context"
 	"log"
 	"log/slog"
 	"net"
 	"os"
 	"path/filepath"
+	"time"
 )
 
 const rsaBits = 4096
@@ -53,3 +55,14 @@ func FileExists(filename string) bool {
 	}
 	return !info.IsDir()
 }
+
+func SleepWithContext(ctx context.Context, d time.Duration) {
+	timer := time.NewTimer(d)
+	select {
+	case <-ctx.Done():
+		if !timer.Stop() {
+			<-timer.C
+		}
+	case <-timer.C:
+	}
+}

main.go

@@ -66,7 +66,9 @@ func main() {
 	}
 
 	// TODO - how to pass this to the other packages that will need this info?
-	s, err := settings.ReadYMLSettings(logger, settingsFile)
+	s := settings.New(logger, settingsFile)
+	err = s.ReadYMLSettings()
+	//s, err := settings.ReadYMLSettings(logger, settingsFile)
 	if err != nil {
 		logger.Error("failed to open yaml settings file", "error", err, "filename", settingsFile)
 		//os.Exit(1)
@@ -165,11 +167,11 @@ func main() {
 	if cronInventoryFrequencyString != "" {
 		cronInvFrequency, err = time.ParseDuration(cronInventoryFrequencyString)
 		if err != nil {
-			slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 3600s", "value", cronInventoryFrequencyString, "error", err)
-			cronInvFrequency = time.Second * 3600
+			slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 7200", "value", cronInventoryFrequencyString, "error", err)
+			cronInvFrequency = time.Second * 7200
 		}
 	} else {
-		cronInvFrequency = time.Second * 3600
+		cronInvFrequency = time.Second * 7200
 	}
 	logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency)