From 1fca81a7b3217993003a50acfc0b26241f7948e3 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Wed, 14 Jan 2026 17:39:48 +1100 Subject: [PATCH] consolidate raw sql queries [CI_SKIP] --- db/helpers.go | 278 +++++++++++++++++ internal/tasks/inventorySnapshots.go | 430 +++------------------------ 2 files changed, 319 insertions(+), 389 deletions(-) diff --git a/db/helpers.go b/db/helpers.go index eafcc53..027b23d 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -19,6 +19,11 @@ type SnapshotTotals struct { DiskTotal float64 `db:"disk_total"` } +type ColumnDef struct { + Name string + Type string +} + // ValidateTableName ensures table identifiers are safe for interpolation. func ValidateTableName(name string) error { if name == "" { @@ -167,3 +172,276 @@ WHERE "IsPresent" = 'TRUE' } return totals, nil } + +// EnsureSnapshotTable creates a snapshot table with the standard schema if it does not exist. +func EnsureSnapshotTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { + if _, err := SafeTableName(tableName); err != nil { + return err + } + + driver := strings.ToLower(dbConn.DriverName()) + var ddl string + switch driver { + case "pgx", "postgres": + ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + "RowId" BIGSERIAL PRIMARY KEY, + "InventoryId" BIGINT, + "Name" TEXT NOT NULL, + "Vcenter" TEXT NOT NULL, + "VmId" TEXT, + "EventKey" TEXT, + "CloudId" TEXT, + "CreationTime" BIGINT, + "DeletionTime" BIGINT, + "ResourcePool" TEXT, + "Datacenter" TEXT, + "Cluster" TEXT, + "Folder" TEXT, + "ProvisionedDisk" REAL, + "VcpuCount" BIGINT, + "RamGB" BIGINT, + "IsTemplate" TEXT, + "PoweredOn" TEXT, + "SrmPlaceholder" TEXT, + "VmUuid" TEXT, + "SnapshotTime" BIGINT NOT NULL, + "IsPresent" TEXT NOT NULL +);`, tableName) + default: + ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + "RowId" INTEGER PRIMARY KEY AUTOINCREMENT, + "InventoryId" BIGINT, + "Name" TEXT NOT NULL, + "Vcenter" TEXT NOT NULL, + "VmId" TEXT, + "EventKey" TEXT, + "CloudId" TEXT, + "CreationTime" BIGINT, + "DeletionTime" BIGINT, + "ResourcePool" TEXT, + "Datacenter" TEXT, + "Cluster" TEXT, + "Folder" TEXT, + "ProvisionedDisk" REAL, + "VcpuCount" BIGINT, + "RamGB" BIGINT, + "IsTemplate" TEXT, + "PoweredOn" TEXT, + "SrmPlaceholder" TEXT, + "VmUuid" TEXT, + "SnapshotTime" BIGINT NOT NULL, + "IsPresent" TEXT NOT NULL +);`, tableName) + } + + _, err := dbConn.ExecContext(ctx, ddl) + return err +} + +// BuildDailySummaryInsert returns the SQL to aggregate hourly snapshots into a daily summary table. +func BuildDailySummaryInsert(tableName string, unionQuery string) (string, error) { + if _, err := SafeTableName(tableName); err != nil { + return "", err + } + insert := fmt.Sprintf(` +WITH snapshots AS ( +%s +) +INSERT INTO %s ( + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", + "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", + "Tin", "Bronze", "Silver", "Gold" +) +SELECT + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", + COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", + "DeletionTime", + ( + SELECT s2."ResourcePool" + FROM snapshots s2 + WHERE s2."VmId" = snapshots."VmId" + AND s2."Vcenter" = snapshots."Vcenter" + AND s2."IsPresent" = 'TRUE' + ORDER BY s2."SnapshotTime" DESC + LIMIT 1 + ) AS "ResourcePool", + "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", + AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" +FROM snapshots +GROUP BY + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; +`, unionQuery, tableName) + return insert, nil +} + +// BuildMonthlySummaryInsert returns the SQL to aggregate daily summaries into a monthly summary table. +func BuildMonthlySummaryInsert(tableName string, unionQuery string) (string, error) { + if _, err := SafeTableName(tableName); err != nil { + return "", err + } + insert := fmt.Sprintf(` +WITH snapshots AS ( +%s +) +INSERT INTO %s ( + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", + "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", + "Tin", "Bronze", "Silver", "Gold" +) +SELECT + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + ( + SELECT s2."ResourcePool" + FROM snapshots s2 + WHERE s2."VmId" = snapshots."VmId" + AND s2."Vcenter" = snapshots."Vcenter" + AND s2."IsPresent" = 'TRUE' + ORDER BY s2."SnapshotTime" DESC + LIMIT 1 + ) AS "ResourcePool", + "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", + AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", + 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) + / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" +FROM snapshots +GROUP BY + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; +`, unionQuery, tableName) + return insert, nil +} + +// EnsureSummaryTable creates a daily/monthly summary table with the standard schema if it does not exist. +func EnsureSummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { + if _, err := SafeTableName(tableName); err != nil { + return err + } + + driver := strings.ToLower(dbConn.DriverName()) + var ddl string + switch driver { + case "pgx", "postgres": + ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + "RowId" BIGSERIAL PRIMARY KEY, + "InventoryId" BIGINT, + "Name" TEXT NOT NULL, + "Vcenter" TEXT NOT NULL, + "VmId" TEXT, + "EventKey" TEXT, + "CloudId" TEXT, + "CreationTime" BIGINT, + "DeletionTime" BIGINT, + "ResourcePool" TEXT, + "Datacenter" TEXT, + "Cluster" TEXT, + "Folder" TEXT, + "ProvisionedDisk" REAL, + "VcpuCount" BIGINT, + "RamGB" BIGINT, + "IsTemplate" TEXT, + "PoweredOn" TEXT, + "SrmPlaceholder" TEXT, + "VmUuid" TEXT, + "SamplesPresent" BIGINT NOT NULL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, + "AvgIsPresent" REAL, + "PoolTinPct" REAL, + "PoolBronzePct" REAL, + "PoolSilverPct" REAL, + "PoolGoldPct" REAL, + "Tin" REAL, + "Bronze" REAL, + "Silver" REAL, + "Gold" REAL +);`, tableName) + default: + ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + "RowId" INTEGER PRIMARY KEY AUTOINCREMENT, + "InventoryId" BIGINT, + "Name" TEXT NOT NULL, + "Vcenter" TEXT NOT NULL, + "VmId" TEXT, + "EventKey" TEXT, + "CloudId" TEXT, + "CreationTime" BIGINT, + "DeletionTime" BIGINT, + "ResourcePool" TEXT, + "Datacenter" TEXT, + "Cluster" TEXT, + "Folder" TEXT, + "ProvisionedDisk" REAL, + "VcpuCount" BIGINT, + "RamGB" BIGINT, + "IsTemplate" TEXT, + "PoweredOn" TEXT, + "SrmPlaceholder" TEXT, + "VmUuid" TEXT, + "SamplesPresent" BIGINT NOT NULL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, + "AvgIsPresent" REAL, + "PoolTinPct" REAL, + "PoolBronzePct" REAL, + "PoolSilverPct" REAL, + "PoolGoldPct" REAL, + "Tin" REAL, + "Bronze" REAL, + "Silver" REAL, + "Gold" REAL +);`, tableName) + } + + _, err := dbConn.ExecContext(ctx, ddl) + return err +} diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index b038daf..d2eec7c 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -145,7 +145,7 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } dbConn := c.Database.DB() - if err := ensureDailySummaryTable(ctx, dbConn, summaryTable); err != nil { + if err := db.EnsureSummaryTable(ctx, dbConn, summaryTable); err != nil { return err } if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { @@ -231,60 +231,10 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } } - insertQuery := fmt.Sprintf(` -WITH snapshots AS ( -%s -) -INSERT INTO %s ( - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", - "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", - "Tin", "Bronze", "Silver", "Gold" -) -SELECT - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", - COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", - "DeletionTime", - ( - SELECT s2."ResourcePool" - FROM snapshots s2 - WHERE s2."VmId" = snapshots."VmId" - AND s2."Vcenter" = snapshots."Vcenter" - AND s2."IsPresent" = 'TRUE' - ORDER BY s2."SnapshotTime" DESC - LIMIT 1 - ) AS "ResourcePool", - "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", - AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" -FROM snapshots -GROUP BY - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; -`, unionQuery, summaryTable) + insertQuery, err := db.BuildDailySummaryInsert(summaryTable, unionQuery) + if err != nil { + return err + } if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate daily inventory", "error", err, "date", dayStart.Format("2006-01-02")) @@ -337,7 +287,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time return err } - if err := ensureMonthlySummaryTable(ctx, dbConn, monthlyTable); err != nil { + if err := db.EnsureSummaryTable(ctx, dbConn, monthlyTable); err != nil { return err } if rowsExist, err := db.TableHasRows(ctx, dbConn, monthlyTable); err != nil { @@ -384,57 +334,10 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time ) } - insertQuery := fmt.Sprintf(` -WITH snapshots AS ( -%s -) -INSERT INTO %s ( - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", - "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", - "Tin", "Bronze", "Silver", "Gold" -) -SELECT - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - ( - SELECT s2."ResourcePool" - FROM snapshots s2 - WHERE s2."VmId" = snapshots."VmId" - AND s2."Vcenter" = snapshots."Vcenter" - AND s2."IsPresent" = 'TRUE' - ORDER BY s2."SnapshotTime" DESC - LIMIT 1 - ) AS "ResourcePool", - "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", - AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", - AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", - AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" -FROM snapshots -GROUP BY - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; -`, unionQuery, monthlyTable) + insertQuery, err := db.BuildMonthlySummaryInsert(monthlyTable, unionQuery) + if err != nil { + return err + } if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01")) @@ -533,62 +436,7 @@ func monthlySummaryTableName(t time.Time) (string, error) { } func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { - driver := strings.ToLower(dbConn.DriverName()) - var ddl string - switch driver { - case "pgx", "postgres": - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" BIGSERIAL PRIMARY KEY, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "SnapshotTime" BIGINT NOT NULL, - "IsPresent" TEXT NOT NULL -);`, tableName) - default: - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" INTEGER PRIMARY KEY AUTOINCREMENT, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "SnapshotTime" BIGINT NOT NULL, - "IsPresent" TEXT NOT NULL -);`, tableName) - } - - if _, err := dbConn.ExecContext(ctx, ddl); err != nil { + if err := db.EnsureSnapshotTable(ctx, dbConn, tableName); err != nil { return err } if err := ensureSnapshotRowID(ctx, dbConn, tableName); err != nil { @@ -600,215 +448,6 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s {Name: "RamGB", Type: "BIGINT"}, }) } - -func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { - driver := strings.ToLower(dbConn.DriverName()) - var ddl string - switch driver { - case "pgx", "postgres": - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" BIGSERIAL PRIMARY KEY, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "SamplesPresent" BIGINT NOT NULL, - "AvgVcpuCount" REAL, - "AvgRamGB" REAL, - "AvgProvisionedDisk" REAL, - "AvgIsPresent" REAL, - "PoolTinPct" REAL, - "PoolBronzePct" REAL, - "PoolSilverPct" REAL, - "PoolGoldPct" REAL, - "Tin" REAL, - "Bronze" REAL, - "Silver" REAL, - "Gold" REAL -);`, tableName) - default: - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" INTEGER PRIMARY KEY AUTOINCREMENT, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "SamplesPresent" BIGINT NOT NULL, - "AvgVcpuCount" REAL, - "AvgRamGB" REAL, - "AvgProvisionedDisk" REAL, - "AvgIsPresent" REAL, - "PoolTinPct" REAL, - "PoolBronzePct" REAL, - "PoolSilverPct" REAL, - "PoolGoldPct" REAL, - "Tin" REAL, - "Bronze" REAL, - "Silver" REAL, - "Gold" REAL -);`, tableName) - } - - if _, err := dbConn.ExecContext(ctx, ddl); err != nil { - return err - } - if err := ensureSnapshotRowID(ctx, dbConn, tableName); err != nil { - return err - } - - if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil { - return err - } - - return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgVcpuCount", Type: "REAL"}, - {Name: "AvgRamGB", Type: "REAL"}, - {Name: "AvgProvisionedDisk", Type: "REAL"}, - {Name: "AvgIsPresent", Type: "REAL"}, - {Name: "PoolTinPct", Type: "REAL"}, - {Name: "PoolBronzePct", Type: "REAL"}, - {Name: "PoolSilverPct", Type: "REAL"}, - {Name: "PoolGoldPct", Type: "REAL"}, - {Name: "Tin", Type: "REAL"}, - {Name: "Bronze", Type: "REAL"}, - {Name: "Silver", Type: "REAL"}, - {Name: "Gold", Type: "REAL"}, - }) -} - -func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { - driver := strings.ToLower(dbConn.DriverName()) - var ddl string - switch driver { - case "pgx", "postgres": - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" BIGSERIAL PRIMARY KEY, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "AvgVcpuCount" REAL, - "AvgRamGB" REAL, - "AvgProvisionedDisk" REAL, - "AvgIsPresent" REAL, - "PoolTinPct" REAL, - "PoolBronzePct" REAL, - "PoolSilverPct" REAL, - "PoolGoldPct" REAL, - "Tin" REAL, - "Bronze" REAL, - "Silver" REAL, - "Gold" REAL -);`, tableName) - default: - ddl = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( - "RowId" INTEGER PRIMARY KEY AUTOINCREMENT, - "InventoryId" BIGINT, - "Name" TEXT NOT NULL, - "Vcenter" TEXT NOT NULL, - "VmId" TEXT, - "EventKey" TEXT, - "CloudId" TEXT, - "CreationTime" BIGINT, - "DeletionTime" BIGINT, - "ResourcePool" TEXT TEXT, - "Datacenter" TEXT, - "Cluster" TEXT, - "Folder" TEXT, - "ProvisionedDisk" REAL, - "VcpuCount" BIGINT, - "RamGB" BIGINT, - "IsTemplate" TEXT, - "PoweredOn" TEXT, - "SrmPlaceholder" TEXT, - "VmUuid" TEXT, - "AvgVcpuCount" REAL, - "AvgRamGB" REAL, - "AvgProvisionedDisk" REAL, - "AvgIsPresent" REAL, - "PoolTinPct" REAL, - "PoolBronzePct" REAL, - "PoolSilverPct" REAL, - "PoolGoldPct" REAL, - "Tin" REAL, - "Bronze" REAL, - "Silver" REAL, - "Gold" REAL -);`, tableName) - } - - if _, err := dbConn.ExecContext(ctx, ddl); err != nil { - return err - } - if err := ensureSnapshotRowID(ctx, dbConn, tableName); err != nil { - return err - } - - if err := ensureSnapshotColumns(ctx, dbConn, tableName, baseSummaryColumns()); err != nil { - return err - } - - return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgVcpuCount", Type: "REAL"}, - {Name: "AvgRamGB", Type: "REAL"}, - {Name: "AvgProvisionedDisk", Type: "REAL"}, - {Name: "AvgIsPresent", Type: "REAL"}, - {Name: "PoolTinPct", Type: "REAL"}, - {Name: "PoolBronzePct", Type: "REAL"}, - {Name: "PoolSilverPct", Type: "REAL"}, - {Name: "PoolGoldPct", Type: "REAL"}, - {Name: "Tin", Type: "REAL"}, - {Name: "Bronze", Type: "REAL"}, - {Name: "Silver", Type: "REAL"}, - {Name: "Gold", Type: "REAL"}, - }) -} - func buildUnionQuery(tables []string, columns []string, whereClause string) string { queries := make([]string, 0, len(tables)) columnList := strings.Join(columns, ", ") @@ -886,19 +525,7 @@ type columnDef struct { Type string } -func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error { - if _, err := db.SafeTableName(tableName); err != nil { - return err - } - for _, column := range columns { - if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil { - return err - } - } - return nil -} - -func baseSummaryColumns() []columnDef { +func summaryMetricColumns() []columnDef { return []columnDef{ {Name: "InventoryId", Type: "BIGINT"}, {Name: "Name", Type: "TEXT"}, @@ -923,6 +550,35 @@ func baseSummaryColumns() []columnDef { } } +func summaryAvgColumns() []columnDef { + return []columnDef{ + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, + {Name: "AvgIsPresent", Type: "REAL"}, + {Name: "PoolTinPct", Type: "REAL"}, + {Name: "PoolBronzePct", Type: "REAL"}, + {Name: "PoolSilverPct", Type: "REAL"}, + {Name: "PoolGoldPct", Type: "REAL"}, + {Name: "Tin", Type: "REAL"}, + {Name: "Bronze", Type: "REAL"}, + {Name: "Silver", Type: "REAL"}, + {Name: "Gold", Type: "REAL"}, + } +} + +func ensureSnapshotColumns(ctx context.Context, dbConn *sqlx.DB, tableName string, columns []columnDef) error { + if _, err := db.SafeTableName(tableName); err != nil { + return err + } + for _, column := range columns { + if err := addColumnIfMissing(ctx, dbConn, tableName, column); err != nil { + return err + } + } + return nil +} + func addColumnIfMissing(ctx context.Context, dbConn *sqlx.DB, tableName string, column columnDef) error { query := fmt.Sprintf(`ALTER TABLE %s ADD COLUMN "%s" %s`, tableName, column.Name, column.Type) if _, err := dbConn.ExecContext(ctx, query); err != nil { @@ -965,10 +621,6 @@ func ensureSnapshotRowID(ctx context.Context, dbConn *sqlx.DB, tableName string) return nil } -func tableExists(ctx context.Context, dbConn *sqlx.DB, table string) bool { - return db.TableExists(ctx, dbConn, table) -} - func nullInt64ToInt(value sql.NullInt64) int64 { if value.Valid { return value.Int64