nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve aggregations
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-15 09:57:05 +11:00
parent 457d9395f0
commit 50e9921955
8 changed files with 261 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -11,6 +11,7 @@
vctp vctp
build/ build/
reports/ reports/
reports/*.xlsx
settings.yaml settings.yaml
# Certificates # Certificates
@@ -44,7 +45,7 @@ appengine-generated/
tmp/ tmp/
pb_data/ pb_data/
# General # Generalis
.DS_Store .DS_Store
.AppleDouble .AppleDouble
.LSOverride .LSOverride

221
db/helpers.go
View File

@@ -281,9 +281,17 @@ func EnsureSnapshotTable(ctx context.Context, dbConn *sqlx.DB, tableName string)
return err return err
} }
index := fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName) indexes := []string{
_, err = dbConn.ExecContext(ctx, index) fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName),
return err fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_snapshottime_idx ON %s ("SnapshotTime")`, tableName, tableName),
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName),
}
for _, idx := range indexes {
if _, err := dbConn.ExecContext(ctx, idx); err != nil {
return err
}
}
return nil
} }
// BackfillSerialColumn sets missing values in a serial-like column for Postgres tables. // BackfillSerialColumn sets missing values in a serial-like column for Postgres tables.
@@ -378,6 +386,20 @@ func BuildDailySummaryInsert(tableName string, unionQuery string) (string, error
insert := fmt.Sprintf(` insert := fmt.Sprintf(`
WITH snapshots AS ( WITH snapshots AS (
%s %s
), ordered AS (
SELECT
s.*,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds
FROM snapshots s
), weighted AS (
SELECT
o.*,
CASE
WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600
ELSE o.interval_seconds
END AS weight_seconds
FROM ordered o
) )
INSERT INTO %s ( INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
@@ -390,12 +412,12 @@ INSERT INTO %s (
SELECT SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
"DeletionTime", COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime",
( (
SELECT s2."ResourcePool" SELECT s2."ResourcePool"
FROM snapshots s2 FROM weighted s2
WHERE s2."VmId" = snapshots."VmId" WHERE s2."VmId" = weighted."VmId"
AND s2."Vcenter" = snapshots."Vcenter" AND s2."Vcenter" = weighted."Vcenter"
AND s2."IsPresent" = 'TRUE' AND s2."IsPresent" = 'TRUE'
ORDER BY s2."SnapshotTime" DESC ORDER BY s2."SnapshotTime" DESC
LIMIT 1 LIMIT 1
@@ -403,29 +425,56 @@ SELECT
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END)
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", ELSE NULL END AS "AvgVcpuCount",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", ELSE NULL END AS "AvgRamGB",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", ELSE NULL END AS "AvgProvisionedDisk",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) CASE WHEN SUM(weight_seconds) > 0
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) ELSE NULL END AS "AvgIsPresent",
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) ELSE NULL END AS "PoolTinPct",
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
FROM snapshots THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolBronzePct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolSilverPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolGoldPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Tin",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Bronze",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Silver",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Gold"
FROM weighted
GROUP BY GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, unionQuery, tableName) `, unionQuery, tableName)
@@ -440,51 +489,95 @@ func BuildMonthlySummaryInsert(tableName string, unionQuery string) (string, err
insert := fmt.Sprintf(` insert := fmt.Sprintf(`
WITH snapshots AS ( WITH snapshots AS (
%s %s
), ordered AS (
SELECT
s.*,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds
FROM snapshots s
), weighted AS (
SELECT
o.*,
CASE
WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600
ELSE o.interval_seconds
END AS weight_seconds
FROM ordered o
) )
INSERT INTO %s ( INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SamplesPresent",
"AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct",
"Tin", "Bronze", "Silver", "Gold" "Tin", "Bronze", "Silver", "Gold"
) )
SELECT SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime",
( (
SELECT s2."ResourcePool" SELECT s2."ResourcePool"
FROM snapshots s2 FROM weighted s2
WHERE s2."VmId" = snapshots."VmId" WHERE s2."VmId" = weighted."VmId"
AND s2."Vcenter" = snapshots."Vcenter" AND s2."Vcenter" = weighted."Vcenter"
AND s2."IsPresent" = 'TRUE' AND s2."IsPresent" = 'TRUE'
ORDER BY s2."SnapshotTime" DESC ORDER BY s2."SnapshotTime" DESC
LIMIT 1 LIMIT 1
) AS "ResourcePool", ) AS "ResourcePool",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END)
AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) ELSE NULL END AS "AvgVcpuCount",
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) ELSE NULL END AS "AvgRamGB",
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) ELSE NULL END AS "AvgProvisionedDisk",
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", CASE WHEN SUM(weight_seconds) > 0
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", ELSE NULL END AS "AvgIsPresent",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" ELSE NULL END AS "PoolTinPct",
FROM snapshots CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolBronzePct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolSilverPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "PoolGoldPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Tin",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Bronze",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Silver",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
ELSE NULL END AS "Gold"
FROM weighted
GROUP BY GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, unionQuery, tableName) `, unionQuery, tableName)
@@ -574,6 +667,18 @@ func EnsureSummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string)
);`, tableName) );`, tableName)
} }
_, err := dbConn.ExecContext(ctx, ddl) if _, err := dbConn.ExecContext(ctx, ddl); err != nil {
return err return err
}
indexes := []string{
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName),
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName),
}
for _, idx := range indexes {
if _, err := dbConn.ExecContext(ctx, idx); err != nil {
return err
}
}
return nil
} }

71
internal/report/snapshots.go
View File

@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@@ -341,6 +342,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
startUnix := start.Unix() startUnix := start.Unix()
endUnix := end.Unix() endUnix := end.Unix()
loc := start.Location()
var rows *sqlx.Rows var rows *sqlx.Rows
var err error var err error
@@ -386,7 +388,7 @@ ORDER BY snapshot_time ASC, table_name ASC
} }
records = append(records, SnapshotRecord{ records = append(records, SnapshotRecord{
TableName: tableName, TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0), SnapshotTime: time.Unix(snapshotTime, 0).In(loc),
SnapshotType: recordType, SnapshotType: recordType,
SnapshotCount: snapshotCnt, SnapshotCount: snapshotCnt,
}) })
@@ -394,6 +396,65 @@ ORDER BY snapshot_time ASC, table_name ASC
return records, rows.Err() return records, rows.Err()
} }
func SnapshotRecordsWithFallback(ctx context.Context, database db.Database, snapshotType, prefix, layout string, start, end time.Time) ([]SnapshotRecord, error) {
records, err := ListSnapshotsByRange(ctx, database, snapshotType, start, end)
if err == nil && len(records) > 0 {
return records, nil
}
fallback, err2 := recordsFromTableNames(ctx, database, snapshotType, prefix, layout, start, end)
if err2 != nil {
if err != nil {
return nil, err
}
return nil, err2
}
if len(fallback) > 0 {
return fallback, nil
}
return records, err
}
func recordsFromTableNames(ctx context.Context, database db.Database, snapshotType, prefix, layout string, start, end time.Time) ([]SnapshotRecord, error) {
tables, err := ListTablesByPrefix(ctx, database, prefix)
if err != nil {
return nil, err
}
records := make([]SnapshotRecord, 0, len(tables))
for _, table := range tables {
if !strings.HasPrefix(table, prefix) {
continue
}
suffix := strings.TrimPrefix(table, prefix)
var ts time.Time
switch layout {
case "epoch":
val, err := strconv.ParseInt(suffix, 10, 64)
if err != nil {
continue
}
ts = time.Unix(val, 0)
default:
parsed, err := time.ParseInLocation(layout, suffix, time.Local)
if err != nil {
continue
}
ts = parsed
}
if !ts.Before(start) && ts.Before(end) {
records = append(records, SnapshotRecord{
TableName: table,
SnapshotTime: ts,
SnapshotType: snapshotType,
})
}
}
sort.Slice(records, func(i, j int) bool {
return records[i].SnapshotTime.Before(records[j].SnapshotTime)
})
return records, nil
}
func LatestSnapshotTime(ctx context.Context, database db.Database, snapshotType string) (time.Time, error) { func LatestSnapshotTime(ctx context.Context, database db.Database, snapshotType string) (time.Time, error) {
dbConn := database.DB() dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName()) driver := strings.ToLower(dbConn.DriverName())
@@ -665,7 +726,7 @@ func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Cont
func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.Context, xlsx *excelize.File, tableName string) { func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.Context, xlsx *excelize.File, tableName string) {
if strings.HasPrefix(tableName, "inventory_daily_summary_") { if strings.HasPrefix(tableName, "inventory_daily_summary_") {
suffix := strings.TrimPrefix(tableName, "inventory_daily_summary_") suffix := strings.TrimPrefix(tableName, "inventory_daily_summary_")
dayStart, err := time.Parse("20060102", suffix) dayStart, err := time.ParseInLocation("20060102", suffix, time.Local)
if err != nil { if err != nil {
return return
} }
@@ -673,7 +734,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
if err := EnsureSnapshotRegistry(ctx, database); err != nil { if err := EnsureSnapshotRegistry(ctx, database); err != nil {
return return
} }
records, err := ListSnapshotsByRange(ctx, database, "hourly", dayStart, dayEnd) records, err := SnapshotRecordsWithFallback(ctx, database, "hourly", "inventory_hourly_", "epoch", dayStart, dayEnd)
if err != nil || len(records) == 0 { if err != nil || len(records) == 0 {
return return
} }
@@ -687,7 +748,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
if strings.HasPrefix(tableName, "inventory_monthly_summary_") { if strings.HasPrefix(tableName, "inventory_monthly_summary_") {
suffix := strings.TrimPrefix(tableName, "inventory_monthly_summary_") suffix := strings.TrimPrefix(tableName, "inventory_monthly_summary_")
monthStart, err := time.Parse("200601", suffix) monthStart, err := time.ParseInLocation("200601", suffix, time.Local)
if err != nil { if err != nil {
return return
} }
@@ -695,7 +756,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
if err := EnsureSnapshotRegistry(ctx, database); err != nil { if err := EnsureSnapshotRegistry(ctx, database); err != nil {
return return
} }
records, err := ListSnapshotsByRange(ctx, database, "daily", monthStart, monthEnd) records, err := SnapshotRecordsWithFallback(ctx, database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd)
if err != nil || len(records) == 0 { if err != nil || len(records) == 0 {
return return
} }

6
internal/tasks/dailyAggregate.go
View File

@@ -52,10 +52,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
} }
} }
hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd) hourlySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", dayStart, dayEnd)
if err != nil { if err != nil {
return err return err
} }
hourlySnapshots = filterRecordsInRange(hourlySnapshots, dayStart, dayEnd)
hourlySnapshots = filterSnapshotsWithRows(ctx, dbConn, hourlySnapshots) hourlySnapshots = filterSnapshotsWithRows(ctx, dbConn, hourlySnapshots)
if len(hourlySnapshots) == 0 { if len(hourlySnapshots) == 0 {
return fmt.Errorf("no hourly snapshot tables found for %s", dayStart.Format("2006-01-02")) return fmt.Errorf("no hourly snapshot tables found for %s", dayStart.Format("2006-01-02"))
@@ -85,8 +86,9 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti
prevStart := dayStart.AddDate(0, 0, -1) prevStart := dayStart.AddDate(0, 0, -1)
prevEnd := dayStart prevEnd := dayStart
prevSnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", prevStart, prevEnd) prevSnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", prevStart, prevEnd)
if err == nil && len(prevSnapshots) > 0 { if err == nil && len(prevSnapshots) > 0 {
prevSnapshots = filterRecordsInRange(prevSnapshots, prevStart, prevEnd)
prevSnapshots = filterSnapshotsWithRows(ctx, dbConn, prevSnapshots) prevSnapshots = filterSnapshotsWithRows(ctx, dbConn, prevSnapshots)
prevTables := make([]string, 0, len(prevSnapshots)) prevTables := make([]string, 0, len(prevSnapshots))
for _, snapshot := range prevSnapshots { for _, snapshot := range prevSnapshots {

31
internal/tasks/inventorySnapshots.go
View File

@@ -370,6 +370,16 @@ func filterSnapshotsWithRows(ctx context.Context, dbConn *sqlx.DB, snapshots []r
return filtered return filtered
} }
func filterRecordsInRange(records []report.SnapshotRecord, start, end time.Time) []report.SnapshotRecord {
filtered := records[:0]
for _, r := range records {
if !r.SnapshotTime.Before(start) && r.SnapshotTime.Before(end) {
filtered = append(filtered, r)
}
}
return filtered
}
type columnDef struct { type columnDef struct {
Name string Name string
Type string Type string
@@ -437,18 +447,17 @@ func normalizeResourcePool(value string) string {
if trimmed == "" { if trimmed == "" {
return "" return ""
} }
switch { lower := strings.ToLower(trimmed)
case strings.EqualFold(trimmed, "tin"): canonical := map[string]string{
return "Tin" "tin": "Tin",
case strings.EqualFold(trimmed, "bronze"): "bronze": "Bronze",
return "Bronze" "silver": "Silver",
case strings.EqualFold(trimmed, "silver"): "gold": "Gold",
return "Silver"
case strings.EqualFold(trimmed, "gold"):
return "Gold"
default:
return trimmed
} }
if val, ok := canonical[lower]; ok {
return val
}
return trimmed
} }
func (c *CronTask) reportsDir() string { func (c *CronTask) reportsDir() string {

3
internal/tasks/monthlyAggregate.go
View File

@@ -35,10 +35,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
monthStart := time.Date(targetMonth.Year(), targetMonth.Month(), 1, 0, 0, 0, 0, targetMonth.Location()) monthStart := time.Date(targetMonth.Year(), targetMonth.Month(), 1, 0, 0, 0, 0, targetMonth.Location())
monthEnd := monthStart.AddDate(0, 1, 0) monthEnd := monthStart.AddDate(0, 1, 0)
dailySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", monthStart, monthEnd) dailySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd)
if err != nil { if err != nil {
return err return err
} }
dailySnapshots = filterRecordsInRange(dailySnapshots, monthStart, monthEnd)
dbConn := c.Database.DB() dbConn := c.Database.DB()
dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots) dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots)

5
server/handler/snapshotAggregate.go
View File

@@ -24,6 +24,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request)
snapshotType := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("type"))) snapshotType := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("type")))
dateValue := strings.TrimSpace(r.URL.Query().Get("date")) dateValue := strings.TrimSpace(r.URL.Query().Get("date"))
startedAt := time.Now() startedAt := time.Now()
loc := time.Now().Location()
if snapshotType == "" || dateValue == "" { if snapshotType == "" || dateValue == "" {
h.Logger.Warn("Snapshot aggregation request missing parameters", h.Logger.Warn("Snapshot aggregation request missing parameters",
@@ -43,7 +44,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request)
switch snapshotType { switch snapshotType {
case "daily": case "daily":
parsed, err := time.Parse("2006-01-02", dateValue) parsed, err := time.ParseInLocation("2006-01-02", dateValue, loc)
if err != nil { if err != nil {
h.Logger.Warn("Snapshot aggregation invalid daily date format", "date", dateValue) h.Logger.Warn("Snapshot aggregation invalid daily date format", "date", dateValue)
writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM-DD") writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM-DD")
@@ -56,7 +57,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request)
return return
} }
case "monthly": case "monthly":
parsed, err := time.Parse("2006-01", dateValue) parsed, err := time.ParseInLocation("2006-01", dateValue, loc)
if err != nil { if err != nil {
h.Logger.Warn("Snapshot aggregation invalid monthly date format", "date", dateValue) h.Logger.Warn("Snapshot aggregation invalid monthly date format", "date", dateValue)
writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM") writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM")

2
src/vctp.service
View File

@@ -1,5 +1,5 @@
[Unit] [Unit]
Description=vCTP monitors VMware VM inventory and event data to build chargeback reports Description=vSphere Chargeback Tracking Platform
Documentation=https://gitlab.dell.com/ Documentation=https://gitlab.dell.com/
ConditionPathExists=/usr/bin/vctp-linux-amd64 ConditionPathExists=/usr/bin/vctp-linux-amd64
After=network.target After=network.target