diff --git a/db/migrations/20240912012927_init.sql b/db/migrations/20240912012927_init.sql index e391e1f..8e81c6c 100644 --- a/db/migrations/20240912012927_init.sql +++ b/db/migrations/20240912012927_init.sql @@ -10,7 +10,6 @@ CREATE TABLE IF NOT EXISTS "Inventory" ( "CreationTime" INTEGER, "DeletionTime" INTEGER, "ResourcePool" TEXT, - "VmType" TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, diff --git a/db/migrations/20250116090000_drop_vmtype.sql b/db/migrations/20250116090000_drop_vmtype.sql new file mode 100644 index 0000000..165dc59 --- /dev/null +++ b/db/migrations/20250116090000_drop_vmtype.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE "Inventory" DROP COLUMN "VmType"; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE "Inventory" ADD COLUMN "VmType" TEXT; +-- +goose StatementEnd diff --git a/db/migrations_postgres/20240912012927_init.sql b/db/migrations_postgres/20240912012927_init.sql index 170c09f..a2112fc 100644 --- a/db/migrations_postgres/20240912012927_init.sql +++ b/db/migrations_postgres/20240912012927_init.sql @@ -10,7 +10,6 @@ CREATE TABLE IF NOT EXISTS "Inventory" ( "CreationTime" BIGINT, "DeletionTime" BIGINT, "ResourcePool" TEXT, - "VmType" TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, diff --git a/db/migrations_postgres/20250116090000_drop_vmtype.sql b/db/migrations_postgres/20250116090000_drop_vmtype.sql new file mode 100644 index 0000000..f9b88ee --- /dev/null +++ b/db/migrations_postgres/20250116090000_drop_vmtype.sql @@ -0,0 +1,9 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE "Inventory" DROP COLUMN IF EXISTS "VmType"; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE "Inventory" ADD COLUMN "VmType" TEXT; +-- +goose StatementEnd diff --git a/db/queries/models.go b/db/queries/models.go index 382821d..9062ca9 100644 --- a/db/queries/models.go +++ b/db/queries/models.go @@ -36,7 +36,6 @@ type Inventory struct { CreationTime sql.NullInt64 `db:"CreationTime" json:"CreationTime"` DeletionTime sql.NullInt64 `db:"DeletionTime" json:"DeletionTime"` ResourcePool sql.NullString `db:"ResourcePool" json:"ResourcePool"` - VmType sql.NullString `db:"VmType" json:"VmType"` Datacenter sql.NullString `db:"Datacenter" json:"Datacenter"` Cluster sql.NullString `db:"Cluster" json:"Cluster"` Folder sql.NullString `db:"Folder" json:"Folder"` diff --git a/db/queries/query.sql b/db/queries/query.sql index 654c348..63542cd 100644 --- a/db/queries/query.sql +++ b/db/queries/query.sql @@ -32,9 +32,9 @@ WHERE "CloudId" = ? LIMIT 1; -- name: CreateInventory :one INSERT INTO inventory ( - "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" + "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" ) VALUES( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) RETURNING *; diff --git a/db/queries/query.sql.go b/db/queries/query.sql.go index 61c01f0..08f0497 100644 --- a/db/queries/query.sql.go +++ b/db/queries/query.sql.go @@ -101,11 +101,11 @@ func (q *Queries) CreateEvent(ctx context.Context, arg CreateEventParams) (Event const createInventory = `-- name: CreateInventory :one INSERT INTO inventory ( - "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "VmType", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" + "Name", "Vcenter", "VmId", "VmUuid", "EventKey", "CloudId", "CreationTime", "ResourcePool", "IsTemplate", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", "InitialRam", "SrmPlaceholder", "PoweredOn" ) VALUES( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` type CreateInventoryParams struct { @@ -117,7 +117,6 @@ type CreateInventoryParams struct { CloudId sql.NullString `db:"CloudId" json:"CloudId"` CreationTime sql.NullInt64 `db:"CreationTime" json:"CreationTime"` ResourcePool sql.NullString `db:"ResourcePool" json:"ResourcePool"` - VmType sql.NullString `db:"VmType" json:"VmType"` IsTemplate interface{} `db:"IsTemplate" json:"IsTemplate"` Datacenter sql.NullString `db:"Datacenter" json:"Datacenter"` Cluster sql.NullString `db:"Cluster" json:"Cluster"` @@ -139,7 +138,6 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams arg.CloudId, arg.CreationTime, arg.ResourcePool, - arg.VmType, arg.IsTemplate, arg.Datacenter, arg.Cluster, @@ -161,7 +159,6 @@ func (q *Queries) CreateInventory(ctx context.Context, arg CreateInventoryParams &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -281,7 +278,7 @@ func (q *Queries) CreateUpdate(ctx context.Context, arg CreateUpdateParams) (Upd } const getInventoryByName = `-- name: GetInventoryByName :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "Name" = ? ` @@ -304,7 +301,6 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -330,7 +326,7 @@ func (q *Queries) GetInventoryByName(ctx context.Context, name string) ([]Invent } const getInventoryByVcenter = `-- name: GetInventoryByVcenter :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "Vcenter" = ? ` @@ -353,7 +349,6 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([] &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -379,7 +374,7 @@ func (q *Queries) GetInventoryByVcenter(ctx context.Context, vcenter string) ([] } const getInventoryEventId = `-- name: GetInventoryEventId :one -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "CloudId" = ? LIMIT 1 ` @@ -396,7 +391,6 @@ func (q *Queries) GetInventoryEventId(ctx context.Context, cloudid sql.NullStrin &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -412,7 +406,7 @@ func (q *Queries) GetInventoryEventId(ctx context.Context, cloudid sql.NullStrin } const getInventoryVcUrl = `-- name: GetInventoryVcUrl :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "Vcenter" = ?1 ` @@ -435,7 +429,6 @@ func (q *Queries) GetInventoryVcUrl(ctx context.Context, vc string) ([]Inventory &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -461,7 +454,7 @@ func (q *Queries) GetInventoryVcUrl(ctx context.Context, vc string) ([]Inventory } const getInventoryVmId = `-- name: GetInventoryVmId :one -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "VmId" = ?1 AND "Datacenter" = ?2 ` @@ -483,7 +476,6 @@ func (q *Queries) GetInventoryVmId(ctx context.Context, arg GetInventoryVmIdPara &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -499,7 +491,7 @@ func (q *Queries) GetInventoryVmId(ctx context.Context, arg GetInventoryVmIdPara } const getInventoryVmUuid = `-- name: GetInventoryVmUuid :one -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory WHERE "VmUuid" = ?1 AND "Datacenter" = ?2 ` @@ -521,7 +513,6 @@ func (q *Queries) GetInventoryVmUuid(ctx context.Context, arg GetInventoryVmUuid &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -537,7 +528,7 @@ func (q *Queries) GetInventoryVmUuid(ctx context.Context, arg GetInventoryVmUuid } const getReportInventory = `-- name: GetReportInventory :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory ORDER BY "CreationTime" ` @@ -560,7 +551,6 @@ func (q *Queries) GetReportInventory(ctx context.Context) ([]Inventory, error) { &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, @@ -679,7 +669,7 @@ func (q *Queries) GetVmUpdates(ctx context.Context, arg GetVmUpdatesParams) ([]U const inventoryCleanup = `-- name: InventoryCleanup :exec DELETE FROM inventory WHERE "VmId" = ?1 AND "Datacenter" = ?2 -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` type InventoryCleanupParams struct { @@ -695,7 +685,7 @@ func (q *Queries) InventoryCleanup(ctx context.Context, arg InventoryCleanupPara const inventoryCleanupTemplates = `-- name: InventoryCleanupTemplates :exec DELETE FROM inventory WHERE "IsTemplate" = 'TRUE' -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` func (q *Queries) InventoryCleanupTemplates(ctx context.Context) error { @@ -706,7 +696,7 @@ func (q *Queries) InventoryCleanupTemplates(ctx context.Context) error { const inventoryCleanupVcenter = `-- name: InventoryCleanupVcenter :exec DELETE FROM inventory WHERE "Vcenter" = ?1 -RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid +RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid ` func (q *Queries) InventoryCleanupVcenter(ctx context.Context, vc string) error { @@ -793,7 +783,7 @@ func (q *Queries) ListEvents(ctx context.Context) ([]Event, error) { } const listInventory = `-- name: ListInventory :many -SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory +SELECT Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid FROM inventory ORDER BY "Name" ` @@ -816,7 +806,6 @@ func (q *Queries) ListInventory(ctx context.Context) ([]Inventory, error) { &i.CreationTime, &i.DeletionTime, &i.ResourcePool, - &i.VmType, &i.Datacenter, &i.Cluster, &i.Folder, diff --git a/db/schema.sql b/db/schema.sql index a646637..25eeebb 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -8,7 +8,6 @@ CREATE TABLE IF NOT EXISTS inventory ( "CreationTime" INTEGER, "DeletionTime" INTEGER, "ResourcePool" TEXT, - "VmType" TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, diff --git a/internal/report/create.go b/internal/report/create.go index c2ca42a..dfb26e6 100644 --- a/internal/report/create.go +++ b/internal/report/create.go @@ -113,12 +113,10 @@ func CreateInventoryReport(logger *slog.Logger, Database db.Database, ctx contex } // Set column autowidth - /* - err = SetColAutoWidth(xlsx, sheetName) - if err != nil { - fmt.Printf("Error setting auto width : '%s'\n", err) - } - */ + err = SetColAutoWidth(xlsx, sheetName) + if err != nil { + logger.Error("Error setting auto width", "error", err) + } // Save the Excel file into a byte buffer if err := xlsx.Write(&buffer); err != nil { @@ -226,12 +224,10 @@ func CreateUpdatesReport(logger *slog.Logger, Database db.Database, ctx context. } // Set column autowidth - /* - err = SetColAutoWidth(xlsx, sheetName) - if err != nil { - fmt.Printf("Error setting auto width : '%s'\n", err) - } - */ + err = SetColAutoWidth(xlsx, sheetName) + if err != nil { + logger.Error("Error setting auto width", "error", err) + } // Save the Excel file into a byte buffer if err := xlsx.Write(&buffer); err != nil { diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index ba9b738..a0e482c 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -412,6 +412,11 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co } dbConn := Database.DB() + if strings.HasPrefix(tableName, "inventory_daily_summary_") || strings.HasPrefix(tableName, "inventory_monthly_summary_") { + if err := ensureSummaryReportColumns(ctx, dbConn, tableName); err != nil { + logger.Warn("Unable to ensure summary columns for report", "error", err, "table", tableName) + } + } columns, err := tableColumns(ctx, dbConn, tableName) if err != nil { return nil, err @@ -424,27 +429,68 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co isDailySummary := strings.HasPrefix(tableName, "inventory_daily_summary_") isMonthlySummary := strings.HasPrefix(tableName, "inventory_monthly_summary_") hideInventoryID := isHourlySnapshot || isDailySummary || isMonthlySummary + hideRowID := isHourlySnapshot || isDailySummary || isMonthlySummary humanizeTimes := isDailySummary || isMonthlySummary + applyTemplateFilter := isHourlySnapshot || isDailySummary || isMonthlySummary type columnSpec struct { Name string SourceIndex int Humanize bool } specs := make([]columnSpec, 0, len(columns)+2) + columnIndex := make(map[string]int, len(columns)) for i, columnName := range columns { + columnIndex[strings.ToLower(columnName)] = i + } + used := make(map[string]struct{}, len(columns)) + addSpec := func(columnName string, sourceIndex int) { if hideInventoryID && strings.EqualFold(columnName, "InventoryId") { - continue + return } - specs = append(specs, columnSpec{Name: columnName, SourceIndex: i}) + if hideRowID && strings.EqualFold(columnName, "RowId") { + return + } + if strings.EqualFold(columnName, "VmType") { + return + } + if (isDailySummary || isMonthlySummary) && (strings.EqualFold(columnName, "EventKey") || strings.EqualFold(columnName, "CloudId")) { + return + } + if (isDailySummary || isMonthlySummary) && strings.EqualFold(columnName, "Gold") { + return + } + specs = append(specs, columnSpec{Name: columnName, SourceIndex: sourceIndex}) if humanizeTimes && columnName == "CreationTime" { - specs = append(specs, columnSpec{Name: "CreationTimeReadable", SourceIndex: i, Humanize: true}) + specs = append(specs, columnSpec{Name: "CreationTimeReadable", SourceIndex: sourceIndex, Humanize: true}) } if humanizeTimes && columnName == "DeletionTime" { - specs = append(specs, columnSpec{Name: "DeletionTimeReadable", SourceIndex: i, Humanize: true}) + specs = append(specs, columnSpec{Name: "DeletionTimeReadable", SourceIndex: sourceIndex, Humanize: true}) + } + } + + if isDailySummary || isMonthlySummary { + for _, columnName := range summaryReportOrder() { + if idx, ok := columnIndex[strings.ToLower(columnName)]; ok { + addSpec(columnName, idx) + used[strings.ToLower(columnName)] = struct{}{} + } else { + logger.Warn("Summary report column missing from table", "table", tableName, "column", columnName) + addSpec(columnName, -1) + } + } + } else { + for i, columnName := range columns { + if _, ok := used[strings.ToLower(columnName)]; ok { + continue + } + addSpec(columnName, i) } } query := fmt.Sprintf(`SELECT * FROM %s`, tableName) + if applyTemplateFilter && hasColumn(columns, "IsTemplate") { + query = fmt.Sprintf(`%s WHERE %s`, query, templateExclusionFilter()) + } orderBy := snapshotOrderBy(columns) if orderBy != "" { query = fmt.Sprintf(`%s ORDER BY "%s" DESC`, query, orderBy) @@ -471,7 +517,11 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co } for i, spec := range specs { - cell := fmt.Sprintf("%s1", string(rune('A'+i))) + cell, err := excelize.CoordinatesToCellName(i+1, 1) + if err != nil { + logger.Error("Error determining header cell", "error", err, "column", i+1) + continue + } xlsx.SetCellValue(sheetName, cell, spec.Name) } @@ -500,7 +550,15 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co return nil, err } for colIndex, spec := range specs { - cell := fmt.Sprintf("%s%d", string(rune('A'+colIndex)), rowIndex) + cell, err := excelize.CoordinatesToCellName(colIndex+1, rowIndex) + if err != nil { + logger.Error("Error determining data cell", "error", err, "column", colIndex+1, "row", rowIndex) + continue + } + if spec.SourceIndex < 0 || spec.SourceIndex >= len(values) { + xlsx.SetCellValue(sheetName, cell, "") + continue + } value := values[spec.SourceIndex] if spec.Humanize { xlsx.SetCellValue(sheetName, cell, formatEpochHuman(value)) @@ -528,8 +586,20 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co logger.Error("Error freezing top row", "error", err) } + if err := SetColAutoWidth(xlsx, sheetName); err != nil { + logger.Error("Error setting auto width", "error", err) + } + + if isDailySummary || isMonthlySummary { + addReportMetadataSheet(logger, xlsx) + } + addTotalsChartSheet(logger, Database, ctx, xlsx, tableName) + if index, err := xlsx.GetSheetIndex(sheetName); err == nil { + xlsx.SetActiveSheet(index) + } + if err := xlsx.Write(&buffer); err != nil { return nil, err } @@ -668,6 +738,133 @@ func snapshotOrderBy(columns []string) string { return "" } +func hasColumn(columns []string, name string) bool { + for _, col := range columns { + if strings.EqualFold(col, name) { + return true + } + } + return false +} + +func templateExclusionFilter() string { + return `COALESCE(CAST("IsTemplate" AS TEXT), '') NOT IN ('TRUE', 'true', '1')` +} + +func quoteSheetName(name string) string { + escaped := strings.ReplaceAll(name, "'", "''") + return fmt.Sprintf("'%s'", escaped) +} + +type columnDef struct { + Name string + Type string +} + +func ensureSummaryReportColumns(ctx context.Context, dbConn *sqlx.DB, tableName string) error { + if err := validateTableName(tableName); err != nil { + return err + } + columns, err := tableColumns(ctx, dbConn, tableName) + if err != nil { + return err + } + existing := make(map[string]struct{}, len(columns)) + for _, col := range columns { + existing[strings.ToLower(col)] = struct{}{} + } + for _, column := range summaryReportColumns() { + if _, ok := existing[strings.ToLower(column.Name)]; ok { + continue + } + query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type) + if _, err := dbConn.ExecContext(ctx, query); err != nil { + return err + } + } + return nil +} + +func summaryReportColumns() []columnDef { + return []columnDef{ + {Name: "InventoryId", Type: "BIGINT"}, + {Name: "Name", Type: "TEXT"}, + {Name: "Vcenter", Type: "TEXT"}, + {Name: "VmId", Type: "TEXT"}, + {Name: "EventKey", Type: "TEXT"}, + {Name: "CloudId", Type: "TEXT"}, + {Name: "CreationTime", Type: "BIGINT"}, + {Name: "DeletionTime", Type: "BIGINT"}, + {Name: "ResourcePool", Type: "TEXT"}, + {Name: "Datacenter", Type: "TEXT"}, + {Name: "Cluster", Type: "TEXT"}, + {Name: "Folder", Type: "TEXT"}, + {Name: "ProvisionedDisk", Type: "REAL"}, + {Name: "VcpuCount", Type: "BIGINT"}, + {Name: "RamGB", Type: "BIGINT"}, + {Name: "IsTemplate", Type: "TEXT"}, + {Name: "PoweredOn", Type: "TEXT"}, + {Name: "SrmPlaceholder", Type: "TEXT"}, + {Name: "VmUuid", Type: "TEXT"}, + {Name: "SamplesPresent", Type: "BIGINT"}, + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, + {Name: "AvgIsPresent", Type: "REAL"}, + {Name: "PoolTinPct", Type: "REAL"}, + {Name: "PoolBronzePct", Type: "REAL"}, + {Name: "PoolSilverPct", Type: "REAL"}, + {Name: "PoolGoldPct", Type: "REAL"}, + {Name: "Tin", Type: "REAL"}, + {Name: "Bronze", Type: "REAL"}, + {Name: "Silver", Type: "REAL"}, + {Name: "Gold", Type: "REAL"}, + } +} + +func summaryReportOrder() []string { + return []string{ + "Name", + "Vcenter", + "VmId", + "CreationTime", + "DeletionTime", + "ResourcePool", + "Datacenter", + "Cluster", + "Folder", + "ProvisionedDisk", + "VcpuCount", + "RamGB", + "IsTemplate", + "PoweredOn", + "SrmPlaceholder", + "VmUuid", + "SamplesPresent", + "AvgVcpuCount", + "AvgRamGB", + "AvgProvisionedDisk", + "AvgIsPresent", + "PoolTinPct", + "PoolBronzePct", + "PoolSilverPct", + "PoolGoldPct", + } +} + +func addReportMetadataSheet(logger *slog.Logger, xlsx *excelize.File) { + sheetName := "Metadata" + if _, err := xlsx.NewSheet(sheetName); err != nil { + logger.Error("Error creating metadata sheet", "error", err) + return + } + xlsx.SetCellValue(sheetName, "A1", "ReportGeneratedAt") + xlsx.SetCellValue(sheetName, "B1", time.Now().Format(time.RFC3339)) + if err := SetColAutoWidth(xlsx, sheetName); err != nil { + logger.Error("Error setting metadata auto width", "error", err) + } +} + func scanRowValues(rows *sqlx.Rows, columnCount int) ([]interface{}, error) { rawValues := make([]interface{}, columnCount) scanArgs := make([]interface{}, columnCount) @@ -722,7 +919,8 @@ SELECT COALESCE(SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END), 0) AS silver_total, COALESCE(SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END), 0) AS gold_total FROM %s -`, record.TableName) +WHERE %s +`, record.TableName, templateExclusionFilter()) var row struct { VmCount int64 `db:"vm_count"` VcpuTotal int64 `db:"vcpu_total"` @@ -768,7 +966,8 @@ SELECT COALESCE(SUM(CASE WHEN "Silver" IS NOT NULL THEN "Silver" ELSE 0 END) / 100.0, 0) AS silver_total, COALESCE(SUM(CASE WHEN "Gold" IS NOT NULL THEN "Gold" ELSE 0 END) / 100.0, 0) AS gold_total FROM %s -`, record.TableName) +WHERE %s +`, record.TableName, templateExclusionFilter()) var row struct { VmCount int64 `db:"vm_count"` VcpuTotal float64 `db:"vcpu_total"` @@ -808,7 +1007,7 @@ func writeTotalsChart(logger *slog.Logger, xlsx *excelize.File, sheetName string } xlsx.SetActiveSheet(index) - headers := []string{"Label", "VmCount", "VcpuCount", "RamGB", "PresenceRatio", "Tin", "Bronze", "Silver", "Gold"} + headers := []string{"Label", "VmCount", "VcpuCount", "RamGB", "ProratedVmCount", "Tin", "Bronze", "Silver", "Gold"} for i, header := range headers { cell, _ := excelize.CoordinatesToCellName(i+1, 1) xlsx.SetCellValue(sheetName, cell, header) @@ -826,26 +1025,47 @@ func writeTotalsChart(logger *slog.Logger, xlsx *excelize.File, sheetName string xlsx.SetCellValue(sheetName, fmt.Sprintf("I%d", row), point.GoldTotal) } - lastRow := len(points) + 1 - series := []excelize.ChartSeries{ - {Name: fmt.Sprintf("%s!$B$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$B$2:$B$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$C$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$C$2:$C$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$D$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$D$2:$D$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$E$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$E$2:$E$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$F$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$F$2:$F$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$G$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$G$2:$G$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$H$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$H$2:$H$%d", sheetName, lastRow)}, - {Name: fmt.Sprintf("%s!$I$1", sheetName), Categories: fmt.Sprintf("%s!$A$2:$A$%d", sheetName, lastRow), Values: fmt.Sprintf("%s!$I$2:$I$%d", sheetName, lastRow)}, + if endCell, err := excelize.CoordinatesToCellName(len(headers), 1); err == nil { + filterRange := "A1:" + endCell + if err := xlsx.AutoFilter(sheetName, filterRange, nil); err != nil { + logger.Error("Error setting totals autofilter", "error", err) + } } - chart := excelize.Chart{ - Type: excelize.Line, - Series: series, - Legend: excelize.ChartLegend{Position: "bottom"}, + if err := SetColAutoWidth(xlsx, sheetName); err != nil { + logger.Error("Error setting totals auto width", "error", err) } - if err := xlsx.AddChart(sheetName, "G2", &chart); err != nil { - logger.Error("Error adding totals chart", "error", err) + lastRow := len(points) + 1 + sheetRef := quoteSheetName(sheetName) + categories := fmt.Sprintf("%s!$A$2:$A$%d", sheetRef, lastRow) + buildSeries := func(col string) excelize.ChartSeries { + return excelize.ChartSeries{ + Name: fmt.Sprintf("%s!$%s$1", sheetRef, col), + Categories: categories, + Values: fmt.Sprintf("%s!$%s$2:$%s$%d", sheetRef, col, col, lastRow), + } } + + makeChart := func(anchor string, cols ...string) { + series := make([]excelize.ChartSeries, 0, len(cols)) + for _, col := range cols { + series = append(series, buildSeries(col)) + } + chart := excelize.Chart{ + Type: excelize.Line, + Series: series, + Legend: excelize.ChartLegend{Position: "bottom"}, + XAxis: excelize.ChartAxis{MajorGridLines: true}, + YAxis: excelize.ChartAxis{MajorGridLines: true}, + } + if err := xlsx.AddChart(sheetName, anchor, &chart); err != nil { + logger.Error("Error adding totals chart", "error", err, "anchor", anchor) + } + } + + makeChart("K2", "B", "E") + makeChart("K18", "C", "D") + makeChart("K34", "F", "G", "H", "I") } func formatEpochHuman(value interface{}) string { diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index e237611..0faaaa1 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -29,7 +29,6 @@ type inventorySnapshotRow struct { CreationTime sql.NullInt64 DeletionTime sql.NullInt64 ResourcePool sql.NullString - VmType sql.NullString Datacenter sql.NullString Cluster sql.NullString Folder sql.NullString @@ -181,10 +180,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } unionQuery := buildUnionQuery(hourlyTables, []string{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, - `"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, + `"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, - `"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`, - }) + `"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`, + }, templateExclusionFilter()) currentTotals, err := snapshotTotalsForUnion(ctx, dbConn, unionQuery) if err != nil { @@ -209,10 +208,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } prevUnion := buildUnionQuery(prevTables, []string{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, - `"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, + `"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, - `"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`, - }) + `"SrmPlaceholder"`, `"VmUuid"`, `"SnapshotTime"`, `"IsPresent"`, + }, templateExclusionFilter()) prevTotals, err := snapshotTotalsForUnion(ctx, dbConn, prevUnion) if err != nil { c.Logger.Warn("unable to calculate previous day totals", "error", err, "date", prevStart.Format("2006-01-02")) @@ -231,15 +230,17 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", "Tin", "Bronze", "Silver", "Gold" ) SELECT - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", + COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", + "DeletionTime", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", @@ -267,7 +268,7 @@ FROM ( ) snapshots GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, summaryTable, unionQuery) @@ -346,10 +347,10 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time } unionQuery := buildUnionQuery(dailyTables, []string{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, - `"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, + `"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, `"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`, - }) + }, templateExclusionFilter()) if strings.TrimSpace(unionQuery) == "" { return fmt.Errorf("no valid daily snapshot tables found for %s", targetMonth.Format("2006-01")) } @@ -370,7 +371,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", @@ -378,7 +379,7 @@ INSERT INTO %s ( ) SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", @@ -405,7 +406,7 @@ FROM ( ) snapshots GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, monthlyTable, unionQuery) @@ -530,8 +531,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -556,8 +556,7 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -601,8 +600,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -638,8 +636,7 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -673,6 +670,10 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str return err } + if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil { + return err + } + return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ {Name: "AvgVcpuCount", Type: "REAL"}, {Name: "AvgRamGB", Type: "REAL"}, @@ -704,8 +705,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -740,8 +740,7 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "CloudId" TEXT, "CreationTime" BIGINT, "DeletionTime" BIGINT, - "ResourcePool" TEXT, - "VmType" TEXT, + "ResourcePool" TEXT TEXT, "Datacenter" TEXT, "Cluster" TEXT, "Folder" TEXT, @@ -774,6 +773,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s return err } + if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil { + return err + } + return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ {Name: "AvgVcpuCount", Type: "REAL"}, {Name: "AvgRamGB", Type: "REAL"}, @@ -790,18 +793,26 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s }) } -func buildUnionQuery(tables []string, columns []string) string { +func buildUnionQuery(tables []string, columns []string, whereClause string) string { queries := make([]string, 0, len(tables)) columnList := strings.Join(columns, ", ") for _, table := range tables { if _, err := safeTableName(table); err != nil { continue } - queries = append(queries, fmt.Sprintf("SELECT %s FROM %s", columnList, table)) + query := fmt.Sprintf("SELECT %s FROM %s", columnList, table) + if whereClause != "" { + query = fmt.Sprintf("%s WHERE %s", query, whereClause) + } + queries = append(queries, query) } return strings.Join(queries, "\nUNION ALL\n") } +func templateExclusionFilter() string { + return `COALESCE(CAST("IsTemplate" AS TEXT), '') NOT IN ('TRUE', 'true', '1')` +} + func parseSnapshotDate(table string, prefix string, layout string) (time.Time, bool) { if !strings.HasPrefix(table, prefix) { return time.Time{}, false @@ -860,10 +871,10 @@ func tableHasRows(ctx context.Context, dbConn *sqlx.DB, table string) (bool, err } type snapshotTotals struct { - VmCount int64 - VcpuTotal int64 - RamTotal int64 - DiskTotal float64 + VmCount int64 `db:"vm_count"` + VcpuTotal int64 `db:"vcpu_total"` + RamTotal int64 `db:"ram_total"` + DiskTotal float64 `db:"disk_total"` } type columnDef struct { @@ -883,6 +894,31 @@ func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName strin return nil } +func baseSummaryColumns() []columnDef { + return []columnDef{ + {Name: "InventoryId", Type: "BIGINT"}, + {Name: "Name", Type: "TEXT"}, + {Name: "Vcenter", Type: "TEXT"}, + {Name: "VmId", Type: "TEXT"}, + {Name: "EventKey", Type: "TEXT"}, + {Name: "CloudId", Type: "TEXT"}, + {Name: "CreationTime", Type: "BIGINT"}, + {Name: "DeletionTime", Type: "BIGINT"}, + {Name: "ResourcePool", Type: "TEXT"}, + {Name: "Datacenter", Type: "TEXT"}, + {Name: "Cluster", Type: "TEXT"}, + {Name: "Folder", Type: "TEXT"}, + {Name: "ProvisionedDisk", Type: "REAL"}, + {Name: "VcpuCount", Type: "BIGINT"}, + {Name: "RamGB", Type: "BIGINT"}, + {Name: "IsTemplate", Type: "TEXT"}, + {Name: "PoweredOn", Type: "TEXT"}, + {Name: "SrmPlaceholder", Type: "TEXT"}, + {Name: "VmUuid", Type: "TEXT"}, + {Name: "SamplesPresent", Type: "BIGINT"}, + } +} + func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error { query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type) if _, err := dbConn.ExecContext(ctx, query); err != nil { @@ -1054,6 +1090,25 @@ func intWithDefault(value int, fallback int) int { return value } +func normalizeResourcePool(value string) string { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return "" + } + switch { + case strings.EqualFold(trimmed, "tin"): + return "Tin" + case strings.EqualFold(trimmed, "bronze"): + return "Bronze" + case strings.EqualFold(trimmed, "silver"): + return "Silver" + case strings.EqualFold(trimmed, "gold"): + return "Gold" + default: + return trimmed + } +} + func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTime time.Time, inv *queries.Inventory) (inventorySnapshotRow, error) { if vmObject == nil { return inventorySnapshotRow{}, fmt.Errorf("missing VM object") @@ -1071,7 +1126,6 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi row.EventKey = inv.EventKey row.CloudId = inv.CloudId row.DeletionTime = inv.DeletionTime - row.VmType = inv.VmType } if vmObject.Config != nil { @@ -1112,7 +1166,9 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi } if inv != nil { - row.ResourcePool = inv.ResourcePool + if inv.ResourcePool.Valid { + row.ResourcePool = sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: true} + } row.Datacenter = inv.Datacenter row.Cluster = inv.Cluster row.Folder = inv.Folder @@ -1144,7 +1200,7 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi if row.ResourcePool.String == "" { if rpName, err := vc.GetVmResourcePool(*vmObject); err == nil { - row.ResourcePool = sql.NullString{String: rpName, Valid: rpName != ""} + row.ResourcePool = sql.NullString{String: normalizeResourcePool(rpName), Valid: rpName != ""} } } @@ -1179,8 +1235,7 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent CloudId: inv.CloudId, CreationTime: inv.CreationTime, DeletionTime: inv.DeletionTime, - ResourcePool: inv.ResourcePool, - VmType: inv.VmType, + ResourcePool: sql.NullString{String: normalizeResourcePool(inv.ResourcePool.String), Valid: inv.ResourcePool.Valid}, Datacenter: inv.Datacenter, Cluster: inv.Cluster, Folder: inv.Folder, @@ -1199,10 +1254,10 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str query := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent" ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); `, tableName) query = sqlx.Rebind(sqlx.BindType(dbConn.DriverName()), query) @@ -1217,7 +1272,6 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); row.CreationTime, row.DeletionTime, row.ResourcePool, - row.VmType, row.Datacenter, row.Cluster, row.Folder, diff --git a/server/handler/snapshotAggregate.go b/server/handler/snapshotAggregate.go index ac03209..2a54fa9 100644 --- a/server/handler/snapshotAggregate.go +++ b/server/handler/snapshotAggregate.go @@ -23,8 +23,13 @@ import ( func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request) { snapshotType := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("type"))) dateValue := strings.TrimSpace(r.URL.Query().Get("date")) + startedAt := time.Now() if snapshotType == "" || dateValue == "" { + h.Logger.Warn("Snapshot aggregation request missing parameters", + "type", snapshotType, + "date", dateValue, + ) writeJSONError(w, http.StatusBadRequest, "type and date are required") return } @@ -40,28 +45,40 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request) case "daily": parsed, err := time.Parse("2006-01-02", dateValue) if err != nil { + h.Logger.Warn("Snapshot aggregation invalid daily date format", "date", dateValue) writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM-DD") return } + h.Logger.Info("Starting daily snapshot aggregation", "date", parsed.Format("2006-01-02"), "force", true) if err := ct.AggregateDailySummary(ctx, parsed, true); err != nil { + h.Logger.Error("Daily snapshot aggregation failed", "date", parsed.Format("2006-01-02"), "error", err) writeJSONError(w, http.StatusInternalServerError, err.Error()) return } case "monthly": parsed, err := time.Parse("2006-01", dateValue) if err != nil { + h.Logger.Warn("Snapshot aggregation invalid monthly date format", "date", dateValue) writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM") return } + h.Logger.Info("Starting monthly snapshot aggregation", "date", parsed.Format("2006-01"), "force", true) if err := ct.AggregateMonthlySummary(ctx, parsed, true); err != nil { + h.Logger.Error("Monthly snapshot aggregation failed", "date", parsed.Format("2006-01"), "error", err) writeJSONError(w, http.StatusInternalServerError, err.Error()) return } default: + h.Logger.Warn("Snapshot aggregation invalid type", "type", snapshotType) writeJSONError(w, http.StatusBadRequest, "type must be daily or monthly") return } + h.Logger.Info("Snapshot aggregation completed", + "type", snapshotType, + "date", dateValue, + "duration", time.Since(startedAt), + ) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{ diff --git a/server/middleware/logging.go b/server/middleware/logging.go index 8d71e13..4d6ba16 100644 --- a/server/middleware/logging.go +++ b/server/middleware/logging.go @@ -25,15 +25,14 @@ func (l *LoggingMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := time.Now() l.handler.ServeHTTP(w, r) - query := r.URL.RawQuery - if query == "" { - query = "-" + requestPath := r.URL.RequestURI() + if requestPath == "" { + requestPath = r.URL.Path } l.logger.Debug( "Request recieved", slog.String("method", r.Method), - slog.String("path", r.URL.Path), - slog.String("query", query), + slog.String("request", requestPath), slog.String("remote", r.RemoteAddr), slog.Duration("duration", time.Since(start)), )