nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfix reports
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-14 13:51:30 +11:00
parent 013ae4568e
commit b9ab34db0a
13 changed files with 404 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
internal/tasks/inventorySnapshots.go
View File

@@ -29,7 +29,6 @@ type inventorySnapshotRow struct {
CreationTime sql.NullInt64
DeletionTime sql.NullInt64
ResourcePool sql.NullString
VmType sql.NullString
Datacenter sql.NullString
Cluster sql.NullString
Folder sql.NullString
@@ -181,10 +180,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
}
unionQuery := buildUnionQuery(hourlyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
currentTotals, err := snapshotTotalsForUnion(ctx, dbConn, unionQuery)
if err != nil {
@@ -209,10 +208,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
}
prevUnion := buildUnionQuery(prevTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
prevTotals, err := snapshotTotalsForUnion(ctx, dbConn, prevUnion)
if err != nil {
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
@@ -231,15 +230,17 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct",
"Tin", "Bronze", "Silver", "Gold"
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
"DeletionTime",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
@@ -267,7 +268,7 @@ FROM (
) snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, summaryTable, unionQuery)
@@ -346,10 +347,10 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
}
unionQuery := buildUnionQuery(dailyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
}, templateExclusionFilter())
if strings.TrimSpace(unionQuery) == "" {
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
}
@@ -370,7 +371,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct",
@@ -378,7 +379,7 @@ INSERT INTO %s (
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB",
@@ -405,7 +406,7 @@ FROM (
) snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, monthlyTable, unionQuery)
@@ -530,8 +531,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -556,8 +556,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -601,8 +600,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -638,8 +636,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -673,6 +670,10 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
return err
}
if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil {
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
@@ -704,8 +705,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -740,8 +740,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -774,6 +773,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
return err
}
if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil {
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
@@ -790,18 +793,26 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
})
}
func buildUnionQuery(tables []string, columns []string) string {
func buildUnionQuery(tables []string, columns []string, whereClause string) string {
queries := make([]string, 0, len(tables))
columnList := strings.Join(columns, ", ")
for _, table := range tables {
if _, err := safeTableName(table); err != nil {
continue
}
queries = append(queries, fmt.Sprintf("SELECT %s FROM %s", columnList, table))
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
}
queries = append(queries, query)
}
return strings.Join(queries, "\nUNION ALL\n")
}
func templateExclusionFilter() string {
return `COALESCE(CAST("IsTemplate" AS TEXT), '') NOT IN ('TRUE', 'true', '1')`
}
func parseSnapshotDate(table string, prefix string, layout string) (time.Time, bool) {
if !strings.HasPrefix(table, prefix) {
return time.Time{}, false
@@ -860,10 +871,10 @@ func tableHasRows(ctx context.Context, dbConn *sqlx.DB, table string) (bool, err
}
type snapshotTotals struct {
VmCount int64
VcpuTotal int64
RamTotal int64
DiskTotal float64
VmCount int64 `db:"vm_count"`
VcpuTotal int64 `db:"vcpu_total"`
RamTotal int64 `db:"ram_total"`
DiskTotal float64 `db:"disk_total"`
}
type columnDef struct {
@@ -883,6 +894,31 @@ func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName strin
return nil
}
func baseSummaryColumns() []columnDef {
return []columnDef{
{Name: "InventoryId", Type: "BIGINT"},
{Name: "Name", Type: "TEXT"},
{Name: "Vcenter", Type: "TEXT"},
{Name: "VmId", Type: "TEXT"},
{Name: "EventKey", Type: "TEXT"},
{Name: "CloudId", Type: "TEXT"},
{Name: "CreationTime", Type: "BIGINT"},
{Name: "DeletionTime", Type: "BIGINT"},
{Name: "ResourcePool", Type: "TEXT"},
{Name: "Datacenter", Type: "TEXT"},
{Name: "Cluster", Type: "TEXT"},
{Name: "Folder", Type: "TEXT"},
{Name: "ProvisionedDisk", Type: "REAL"},
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
{Name: "IsTemplate", Type: "TEXT"},
{Name: "PoweredOn", Type: "TEXT"},
{Name: "SrmPlaceholder", Type: "TEXT"},
{Name: "VmUuid", Type: "TEXT"},
{Name: "SamplesPresent", Type: "BIGINT"},
}
}
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
@@ -1054,6 +1090,25 @@ func intWithDefault(value int, fallback int) int {
return value
}
func normalizeResourcePool(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
switch {
case strings.EqualFold(trimmed, "tin"):
return "Tin"
case strings.EqualFold(trimmed, "bronze"):
return "Bronze"
case strings.EqualFold(trimmed, "silver"):
return "Silver"
case strings.EqualFold(trimmed, "gold"):
return "Gold"
default:
return trimmed
}
}
func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTime time.Time, inv *queries.Inventory) (inventorySnapshotRow, error) {
if vmObject == nil {
return inventorySnapshotRow{}, fmt.Errorf("missing VM object")
@@ -1071,7 +1126,6 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
row.EventKey = inv.EventKey
row.CloudId = inv.CloudId
row.DeletionTime = inv.DeletionTime
row.VmType = inv.VmType
}
if vmObject.Config != nil {
@@ -1112,7 +1166,9 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
}
if inv != nil {
row.ResourcePool = inv.ResourcePool
if inv.ResourcePool.Valid {
row.ResourcePool = sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: true}
}
row.Datacenter = inv.Datacenter
row.Cluster = inv.Cluster
row.Folder = inv.Folder
@@ -1144,7 +1200,7 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
if row.ResourcePool.String == "" {
if rpName, err := vc.GetVmResourcePool(*vmObject); err == nil {
row.ResourcePool = sql.NullString{String: rpName, Valid: rpName != ""}
row.ResourcePool = sql.NullString{String: normalizeResourcePool(rpName), Valid: rpName != ""}
}
}
@@ -1179,8 +1235,7 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
CloudId: inv.CloudId,
CreationTime: inv.CreationTime,
DeletionTime: inv.DeletionTime,
ResourcePool: inv.ResourcePool,
VmType: inv.VmType,
ResourcePool: sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: inv.ResourcePool.Valid},
Datacenter: inv.Datacenter,
Cluster: inv.Cluster,
Folder: inv.Folder,
@@ -1199,10 +1254,10 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str
query := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`, tableName)
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
@@ -1217,7 +1272,6 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
row.CreationTime,
row.DeletionTime,
row.ResourcePool,
row.VmType,
row.Datacenter,
row.Cluster,
row.Folder,