nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfix reports
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-14 13:51:30 +11:00
parent 013ae4568e
commit b9ab34db0a
13 changed files with 404 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
internal/report/create.go
View File

@@ -113,12 +113,10 @@ func CreateInventoryReport(logger *slog.Logger, Database db.Database, ctx contex
}
// Set column autowidth
/*
err = SetColAutoWidth(xlsx, sheetName)
if err != nil {
fmt.Printf("Error setting auto width : '%s'\n", err)
}
*/
err = SetColAutoWidth(xlsx, sheetName)
if err != nil {
logger.Error("Error setting auto width", "error", err)
}
// Save the Excel file into a byte buffer
if err := xlsx.Write(&buffer); err != nil {
@@ -226,12 +224,10 @@ func CreateUpdatesReport(logger *slog.Logger, Database db.Database, ctx context.
}
// Set column autowidth
/*
err = SetColAutoWidth(xlsx, sheetName)
if err != nil {
fmt.Printf("Error setting auto width : '%s'\n", err)
}
*/
err = SetColAutoWidth(xlsx, sheetName)
if err != nil {
logger.Error("Error setting auto width", "error", err)
}
// Save the Excel file into a byte buffer
if err := xlsx.Write(&buffer); err != nil {

270
internal/report/snapshots.go
View File

@@ -412,6 +412,11 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
}
dbConn := Database.DB()
if strings.HasPrefix(tableName, "inventory_daily_summary_") || strings.HasPrefix(tableName, "inventory_monthly_summary_") {
if err := ensureSummaryReportColumns(ctx, dbConn, tableName); err != nil {
logger.Warn("Unable to ensure summary columns for report", "error", err, "table", tableName)
}
}
columns, err := tableColumns(ctx, dbConn, tableName)
if err != nil {
return nil, err
@@ -424,27 +429,68 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
isDailySummary := strings.HasPrefix(tableName, "inventory_daily_summary_")
isMonthlySummary := strings.HasPrefix(tableName, "inventory_monthly_summary_")
hideInventoryID := isHourlySnapshot || isDailySummary || isMonthlySummary
hideRowID := isHourlySnapshot || isDailySummary || isMonthlySummary
humanizeTimes := isDailySummary || isMonthlySummary
applyTemplateFilter := isHourlySnapshot || isDailySummary || isMonthlySummary
type columnSpec struct {
Name string
SourceIndex int
Humanize bool
}
specs := make([]columnSpec, 0, len(columns)+2)
columnIndex := make(map[string]int, len(columns))
for i, columnName := range columns {
columnIndex[strings.ToLower(columnName)] = i
}
used := make(map[string]struct{}, len(columns))
addSpec := func(columnName string, sourceIndex int) {
if hideInventoryID && strings.EqualFold(columnName, "InventoryId") {
continue
return
}
specs = append(specs, columnSpec{Name: columnName, SourceIndex: i})
if hideRowID && strings.EqualFold(columnName, "RowId") {
return
}
if strings.EqualFold(columnName, "VmType") {
return
}
if (isDailySummary || isMonthlySummary) && (strings.EqualFold(columnName, "EventKey") || strings.EqualFold(columnName, "CloudId")) {
return
}
if (isDailySummary || isMonthlySummary) && strings.EqualFold(columnName, "Gold") {
return
}
specs = append(specs, columnSpec{Name: columnName, SourceIndex: sourceIndex})
if humanizeTimes && columnName == "CreationTime" {
specs = append(specs, columnSpec{Name: "CreationTimeReadable", SourceIndex: i, Humanize: true})
specs = append(specs, columnSpec{Name: "CreationTimeReadable", SourceIndex: sourceIndex, Humanize: true})
}
if humanizeTimes && columnName == "DeletionTime" {
specs = append(specs, columnSpec{Name: "DeletionTimeReadable", SourceIndex: i, Humanize: true})
specs = append(specs, columnSpec{Name: "DeletionTimeReadable", SourceIndex: sourceIndex, Humanize: true})
}
}
if isDailySummary || isMonthlySummary {
for _, columnName := range summaryReportOrder() {
if idx, ok := columnIndex[strings.ToLower(columnName)]; ok {
addSpec(columnName, idx)
used[strings.ToLower(columnName)] = struct{}{}
} else {
logger.Warn("Summary report column missing from table", "table", tableName, "column", columnName)
addSpec(columnName, -1)
}
}
} else {
for i, columnName := range columns {
if _, ok := used[strings.ToLower(columnName)]; ok {
continue
}
addSpec(columnName, i)
}
}
query := fmt.Sprintf(`SELECT * FROM %s`, tableName)
if applyTemplateFilter && hasColumn(columns, "IsTemplate") {
query = fmt.Sprintf(`%s WHERE %s`, query, templateExclusionFilter())
}
orderBy := snapshotOrderBy(columns)
if orderBy != "" {
query = fmt.Sprintf(`%s ORDER BY "%s" DESC`, query, orderBy)
@@ -471,7 +517,11 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
}
for i, spec := range specs {
cell := fmt.Sprintf("%s1", string(rune('A'+i)))
cell, err := excelize.CoordinatesToCellName(i+1, 1)
if err != nil {
logger.Error("Error determining header cell", "error", err, "column", i+1)
continue
}
xlsx.SetCellValue(sheetName, cell, spec.Name)
}
@@ -500,7 +550,15 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
return nil, err
}
for colIndex, spec := range specs {
cell := fmt.Sprintf("%s%d", string(rune('A'+colIndex)), rowIndex)
cell, err := excelize.CoordinatesToCellName(colIndex+1, rowIndex)
if err != nil {
logger.Error("Error determining data cell", "error", err, "column", colIndex+1, "row", rowIndex)
continue
}
if spec.SourceIndex < 0 || spec.SourceIndex >= len(values) {
xlsx.SetCellValue(sheetName, cell, "")
continue
}
value := values[spec.SourceIndex]
if spec.Humanize {
xlsx.SetCellValue(sheetName, cell, formatEpochHuman(value))
@@ -528,8 +586,20 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
logger.Error("Error freezing top row", "error", err)
}
if err := SetColAutoWidth(xlsx, sheetName); err != nil {
logger.Error("Error setting auto width", "error", err)
}
if isDailySummary || isMonthlySummary {
addReportMetadataSheet(logger, xlsx)
}
addTotalsChartSheet(logger, Database, ctx, xlsx, tableName)
if index, err := xlsx.GetSheetIndex(sheetName); err == nil {
xlsx.SetActiveSheet(index)
}
if err := xlsx.Write(&buffer); err != nil {
return nil, err
}
@@ -668,6 +738,133 @@ func snapshotOrderBy(columns []string) string {
return ""
}
func hasColumn(columns []string, name string) bool {
for _, col := range columns {
if strings.EqualFold(col, name) {
return true
}
}
return false
}
func templateExclusionFilter() string {
return `COALESCE(CAST("IsTemplate" AS TEXT), '') NOT IN ('TRUE', 'true', '1')`
}
func quoteSheetName(name string) string {
escaped := strings.ReplaceAll(name, "'", "''")
return fmt.Sprintf("'%s'", escaped)
}
type columnDef struct {
Name string
Type string
}
func ensureSummaryReportColumns(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
if err := validateTableName(tableName); err != nil {
return err
}
columns, err := tableColumns(ctx, dbConn, tableName)
if err != nil {
return err
}
existing := make(map[string]struct{}, len(columns))
for _, col := range columns {
existing[strings.ToLower(col)] = struct{}{}
}
for _, column := range summaryReportColumns() {
if _, ok := existing[strings.ToLower(column.Name)]; ok {
continue
}
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
return err
}
}
return nil
}
func summaryReportColumns() []columnDef {
return []columnDef{
{Name: "InventoryId", Type: "BIGINT"},
{Name: "Name", Type: "TEXT"},
{Name: "Vcenter", Type: "TEXT"},
{Name: "VmId", Type: "TEXT"},
{Name: "EventKey", Type: "TEXT"},
{Name: "CloudId", Type: "TEXT"},
{Name: "CreationTime", Type: "BIGINT"},
{Name: "DeletionTime", Type: "BIGINT"},
{Name: "ResourcePool", Type: "TEXT"},
{Name: "Datacenter", Type: "TEXT"},
{Name: "Cluster", Type: "TEXT"},
{Name: "Folder", Type: "TEXT"},
{Name: "ProvisionedDisk", Type: "REAL"},
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
{Name: "IsTemplate", Type: "TEXT"},
{Name: "PoweredOn", Type: "TEXT"},
{Name: "SrmPlaceholder", Type: "TEXT"},
{Name: "VmUuid", Type: "TEXT"},
{Name: "SamplesPresent", Type: "BIGINT"},
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
{Name: "AvgProvisionedDisk", Type: "REAL"},
{Name: "AvgIsPresent", Type: "REAL"},
{Name: "PoolTinPct", Type: "REAL"},
{Name: "PoolBronzePct", Type: "REAL"},
{Name: "PoolSilverPct", Type: "REAL"},
{Name: "PoolGoldPct", Type: "REAL"},
{Name: "Tin", Type: "REAL"},
{Name: "Bronze", Type: "REAL"},
{Name: "Silver", Type: "REAL"},
{Name: "Gold", Type: "REAL"},
}
}
func summaryReportOrder() []string {
return []string{
"Name",
"Vcenter",
"VmId",
"CreationTime",
"DeletionTime",
"ResourcePool",
"Datacenter",
"Cluster",
"Folder",
"ProvisionedDisk",
"VcpuCount",
"RamGB",
"IsTemplate",
"PoweredOn",
"SrmPlaceholder",
"VmUuid",
"SamplesPresent",
"AvgVcpuCount",
"AvgRamGB",
"AvgProvisionedDisk",
"AvgIsPresent",
"PoolTinPct",
"PoolBronzePct",
"PoolSilverPct",
"PoolGoldPct",
}
}
func addReportMetadataSheet(logger *slog.Logger, xlsx *excelize.File) {
sheetName := "Metadata"
if _, err := xlsx.NewSheet(sheetName); err != nil {
logger.Error("Error creating metadata sheet", "error", err)
return
}
xlsx.SetCellValue(sheetName, "A1", "ReportGeneratedAt")
xlsx.SetCellValue(sheetName, "B1", time.Now().Format(time.RFC3339))
if err := SetColAutoWidth(xlsx, sheetName); err != nil {
logger.Error("Error setting metadata auto width", "error", err)
}
}
func scanRowValues(rows *sqlx.Rows, columnCount int) ([]interface{}, error) {
rawValues := make([]interface{}, columnCount)
scanArgs := make([]interface{}, columnCount)
@@ -722,7 +919,8 @@ SELECT
COALESCE(SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END), 0) AS silver_total,
COALESCE(SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END), 0) AS gold_total
FROM %s
`, record.TableName)
WHERE %s
`, record.TableName, templateExclusionFilter())
var row struct {
VmCount int64 `db:"vm_count"`
VcpuTotal int64 `db:"vcpu_total"`
@@ -768,7 +966,8 @@ SELECT
COALESCE(SUM(CASE WHEN "Silver" IS NOT NULL THEN "Silver" ELSE 0 END) / 100.0, 0) AS silver_total,
COALESCE(SUM(CASE WHEN "Gold" IS NOT NULL THEN "Gold" ELSE 0 END) / 100.0, 0) AS gold_total
FROM %s
`, record.TableName)
WHERE %s
`, record.TableName, templateExclusionFilter())
var row struct {
VmCount int64 `db:"vm_count"`
VcpuTotal float64 `db:"vcpu_total"`
@@ -808,7 +1007,7 @@ func writeTotalsChart(logger *slog.Logger, xlsx *excelize.File, sheetName string
}
xlsx.SetActiveSheet(index)
headers := []string{"Label", "VmCount", "VcpuCount", "RamGB", "PresenceRatio", "Tin", "Bronze", "Silver", "Gold"}
headers := []string{"Label", "VmCount", "VcpuCount", "RamGB", "ProratedVmCount", "Tin", "Bronze", "Silver", "Gold"}
for i, header := range headers {
cell, _ := excelize.CoordinatesToCellName(i+1, 1)
xlsx.SetCellValue(sheetName, cell, header)
@@ -826,26 +1025,47 @@ func writeTotalsChart(logger *slog.Logger, xlsx *excelize.File, sheetName string
xlsx.SetCellValue(sheetName, fmt.Sprintf("I%d", row), point.GoldTotal)
}
lastRow := len(points) + 1
series := []excelize.ChartSeries{
{Name: fmt.Sprintf("%s!$B$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$B$2:$B$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$C$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$C$2:$C$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$D$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$D$2:$D$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$E$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$E$2:$E$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$F$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$F$2:$F$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$G$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$G$2:$G$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$H$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$H$2:$H$%d", sheetName, lastRow)},
{Name: fmt.Sprintf("%s!$I$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$I$2:$I$%d", sheetName, lastRow)},
if endCell, err := excelize.CoordinatesToCellName(len(headers), 1); err == nil {
filterRange := "A1:" + endCell
if err := xlsx.AutoFilter(sheetName, filterRange, nil); err != nil {
logger.Error("Error setting totals autofilter", "error", err)
}
}
chart := excelize.Chart{
Type: excelize.Line,
Series: series,
Legend: excelize.ChartLegend{Position: "bottom"},
if err := SetColAutoWidth(xlsx, sheetName); err != nil {
logger.Error("Error setting totals auto width", "error", err)
}
if err := xlsx.AddChart(sheetName, "G2", &chart); err != nil {
logger.Error("Error adding totals chart", "error", err)
lastRow := len(points) + 1
sheetRef := quoteSheetName(sheetName)
categories := fmt.Sprintf("%s!$A$2:$A$%d", sheetRef, lastRow)
buildSeries := func(col string) excelize.ChartSeries {
return excelize.ChartSeries{
Name: fmt.Sprintf("%s!$%s$1", sheetRef, col),
Categories: categories,
Values: fmt.Sprintf("%s!$%s$2:$%s$%d", sheetRef, col, col, lastRow),
}
}
makeChart := func(anchor string, cols ...string) {
series := make([]excelize.ChartSeries, 0, len(cols))
for _, col := range cols {
series = append(series, buildSeries(col))
}
chart := excelize.Chart{
Type: excelize.Line,
Series: series,
Legend: excelize.ChartLegend{Position: "bottom"},
XAxis: excelize.ChartAxis{MajorGridLines: true},
YAxis: excelize.ChartAxis{MajorGridLines: true},
}
if err := xlsx.AddChart(sheetName, anchor, &chart); err != nil {
logger.Error("Error adding totals chart", "error", err, "anchor", anchor)
}
}
makeChart("K2", "B", "E")
makeChart("K18", "C", "D")
makeChart("K34", "F", "G", "H", "I")
}
func formatEpochHuman(value interface{}) string {

138
internal/tasks/inventorySnapshots.go
View File

@@ -29,7 +29,6 @@ type inventorySnapshotRow struct {
CreationTime sql.NullInt64
DeletionTime sql.NullInt64
ResourcePool sql.NullString
VmType sql.NullString
Datacenter sql.NullString
Cluster sql.NullString
Folder sql.NullString
@@ -181,10 +180,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
}
unionQuery := buildUnionQuery(hourlyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
currentTotals, err := snapshotTotalsForUnion(ctx, dbConn, unionQuery)
if err != nil {
@@ -209,10 +208,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
}
prevUnion := buildUnionQuery(prevTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
`"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`,
}, templateExclusionFilter())
prevTotals, err := snapshotTotalsForUnion(ctx, dbConn, prevUnion)
if err != nil {
c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02"))
@@ -231,15 +230,17 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct",
"Tin", "Bronze", "Silver", "Gold"
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
"DeletionTime",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
@@ -267,7 +268,7 @@ FROM (
) snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, summaryTable, unionQuery)
@@ -346,10 +347,10 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
}
unionQuery := buildUnionQuery(dailyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
}, templateExclusionFilter())
if strings.TrimSpace(unionQuery) == "" {
return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01"))
}
@@ -370,7 +371,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct",
@@ -378,7 +379,7 @@ INSERT INTO %s (
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB",
@@ -405,7 +406,7 @@ FROM (
) snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, monthlyTable, unionQuery)
@@ -530,8 +531,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -556,8 +556,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -601,8 +600,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -638,8 +636,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -673,6 +670,10 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
return err
}
if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil {
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
@@ -704,8 +705,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -740,8 +740,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"CloudId" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"VmType" TEXT,
"ResourcePool" TEXT TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
@@ -774,6 +773,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
return err
}
if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil {
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
@@ -790,18 +793,26 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
})
}
func buildUnionQuery(tables []string, columns []string) string {
func buildUnionQuery(tables []string, columns []string, whereClause string) string {
queries := make([]string, 0, len(tables))
columnList := strings.Join(columns, ", ")
for _, table := range tables {
if _, err := safeTableName(table); err != nil {
continue
}
queries = append(queries, fmt.Sprintf("SELECT %s FROM %s", columnList, table))
query := fmt.Sprintf("SELECT %s FROM %s", columnList, table)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
}
queries = append(queries, query)
}
return strings.Join(queries, "\nUNION ALL\n")
}
func templateExclusionFilter() string {
return `COALESCE(CAST("IsTemplate" AS TEXT), '') NOT IN ('TRUE', 'true', '1')`
}
func parseSnapshotDate(table string, prefix string, layout string) (time.Time, bool) {
if !strings.HasPrefix(table, prefix) {
return time.Time{}, false
@@ -860,10 +871,10 @@ func tableHasRows(ctx context.Context, dbConn *sqlx.DB, table string) (bool, err
}
type snapshotTotals struct {
VmCount int64
VcpuTotal int64
RamTotal int64
DiskTotal float64
VmCount int64 `db:"vm_count"`
VcpuTotal int64 `db:"vcpu_total"`
RamTotal int64 `db:"ram_total"`
DiskTotal float64 `db:"disk_total"`
}
type columnDef struct {
@@ -883,6 +894,31 @@ func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName strin
return nil
}
func baseSummaryColumns() []columnDef {
return []columnDef{
{Name: "InventoryId", Type: "BIGINT"},
{Name: "Name", Type: "TEXT"},
{Name: "Vcenter", Type: "TEXT"},
{Name: "VmId", Type: "TEXT"},
{Name: "EventKey", Type: "TEXT"},
{Name: "CloudId", Type: "TEXT"},
{Name: "CreationTime", Type: "BIGINT"},
{Name: "DeletionTime", Type: "BIGINT"},
{Name: "ResourcePool", Type: "TEXT"},
{Name: "Datacenter", Type: "TEXT"},
{Name: "Cluster", Type: "TEXT"},
{Name: "Folder", Type: "TEXT"},
{Name: "ProvisionedDisk", Type: "REAL"},
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
{Name: "IsTemplate", Type: "TEXT"},
{Name: "PoweredOn", Type: "TEXT"},
{Name: "SrmPlaceholder", Type: "TEXT"},
{Name: "VmUuid", Type: "TEXT"},
{Name: "SamplesPresent", Type: "BIGINT"},
}
}
func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error {
query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type)
if _, err := dbConn.ExecContext(ctx, query); err != nil {
@@ -1054,6 +1090,25 @@ func intWithDefault(value int, fallback int) int {
return value
}
func normalizeResourcePool(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
switch {
case strings.EqualFold(trimmed, "tin"):
return "Tin"
case strings.EqualFold(trimmed, "bronze"):
return "Bronze"
case strings.EqualFold(trimmed, "silver"):
return "Silver"
case strings.EqualFold(trimmed, "gold"):
return "Gold"
default:
return trimmed
}
}
func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTime time.Time, inv *queries.Inventory) (inventorySnapshotRow, error) {
if vmObject == nil {
return inventorySnapshotRow{}, fmt.Errorf("missing VM object")
@@ -1071,7 +1126,6 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
row.EventKey = inv.EventKey
row.CloudId = inv.CloudId
row.DeletionTime = inv.DeletionTime
row.VmType = inv.VmType
}
if vmObject.Config != nil {
@@ -1112,7 +1166,9 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
}
if inv != nil {
row.ResourcePool = inv.ResourcePool
if inv.ResourcePool.Valid {
row.ResourcePool = sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: true}
}
row.Datacenter = inv.Datacenter
row.Cluster = inv.Cluster
row.Folder = inv.Folder
@@ -1144,7 +1200,7 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
if row.ResourcePool.String == "" {
if rpName, err := vc.GetVmResourcePool(*vmObject); err == nil {
row.ResourcePool = sql.NullString{String: rpName, Valid: rpName != ""}
row.ResourcePool = sql.NullString{String: normalizeResourcePool(rpName), Valid: rpName != ""}
}
}
@@ -1179,8 +1235,7 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
CloudId: inv.CloudId,
CreationTime: inv.CreationTime,
DeletionTime: inv.DeletionTime,
ResourcePool: inv.ResourcePool,
VmType: inv.VmType,
ResourcePool: sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: inv.ResourcePool.Valid},
Datacenter: inv.Datacenter,
Cluster: inv.Cluster,
Folder: inv.Folder,
@@ -1199,10 +1254,10 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str
query := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`, tableName)
query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query)
@@ -1217,7 +1272,6 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
row.CreationTime,
row.DeletionTime,
row.ResourcePool,
row.VmType,
row.Datacenter,
row.Cluster,
row.Folder,