fix averages in aggregation jobs
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-15 10:57:27 +11:00
parent 50e9921955
commit 11f7d36bfc
2 changed files with 114 additions and 122 deletions

View File

@@ -386,20 +386,31 @@ func BuildDailySummaryInsert(tableName string, unionQuery string) (string, error
insert := fmt.Sprintf(`
WITH snapshots AS (
%s
), ordered AS (
), totals AS (
SELECT COUNT(DISTINCT "SnapshotTime") AS total_samples FROM snapshots
), agg AS (
SELECT
s.*,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds
FROM snapshots s
), weighted AS (
SELECT
o.*,
CASE
WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600
ELSE o.interval_seconds
END AS weight_seconds
FROM ordered o
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
MIN(NULLIF("CreationTime", 0)) AS any_creation,
MAX(NULLIF("DeletionTime", 0)) AS any_deletion,
MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END) AS first_present,
MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END) AS last_present,
MAX(CASE WHEN "IsPresent" = 'FALSE' THEN "SnapshotTime" END) AS last_absent,
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS samples_present,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END) AS sum_vcpu,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END) AS sum_ram,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END) AS sum_disk,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) AS tin_hits,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) AS bronze_hits,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) AS silver_hits,
SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) AS gold_hits
FROM snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"
)
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
@@ -410,73 +421,69 @@ INSERT INTO %s (
"Tin", "Bronze", "Silver", "Gold"
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime",
agg."InventoryId", agg."Name", agg."Vcenter", agg."VmId", agg."EventKey", agg."CloudId",
COALESCE(agg.any_creation, agg.first_present, 0) AS "CreationTime",
CASE
WHEN agg.last_present IS NULL THEN NULLIF(agg.any_deletion, 0)
WHEN agg.last_absent IS NOT NULL AND agg.last_absent > agg.last_present THEN agg.last_absent
ELSE NULLIF(agg.any_deletion, 0)
END AS "DeletionTime",
(
SELECT s2."ResourcePool"
FROM weighted s2
WHERE s2."VmId" = weighted."VmId"
AND s2."Vcenter" = weighted."Vcenter"
FROM snapshots s2
WHERE s2."VmId" = agg."VmId"
AND s2."Vcenter" = agg."Vcenter"
AND s2."IsPresent" = 'TRUE'
ORDER BY s2."SnapshotTime" DESC
LIMIT 1
) AS "ResourcePool",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
agg."Datacenter", agg."Cluster", agg."Folder", agg."ProvisionedDisk", agg."VcpuCount",
agg."RamGB", agg."IsTemplate", agg."PoweredOn", agg."SrmPlaceholder", agg."VmUuid",
agg.samples_present AS "SamplesPresent",
CASE WHEN totals.total_samples > 0
THEN 1.0 * agg.sum_vcpu / totals.total_samples
ELSE NULL END AS "AvgVcpuCount",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 1.0 * agg.sum_ram / totals.total_samples
ELSE NULL END AS "AvgRamGB",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 1.0 * agg.sum_disk / totals.total_samples
ELSE NULL END AS "AvgProvisionedDisk",
CASE WHEN SUM(weight_seconds) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds)
CASE WHEN totals.total_samples > 0
THEN 1.0 * agg.samples_present / totals.total_samples
ELSE NULL END AS "AvgIsPresent",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.tin_hits / agg.samples_present
ELSE NULL END AS "PoolTinPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.bronze_hits / agg.samples_present
ELSE NULL END AS "PoolBronzePct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.silver_hits / agg.samples_present
ELSE NULL END AS "PoolSilverPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.gold_hits / agg.samples_present
ELSE NULL END AS "PoolGoldPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.tin_hits / agg.samples_present
ELSE NULL END AS "Tin",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.bronze_hits / agg.samples_present
ELSE NULL END AS "Bronze",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.silver_hits / agg.samples_present
ELSE NULL END AS "Silver",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN agg.samples_present > 0
THEN 100.0 * agg.gold_hits / agg.samples_present
ELSE NULL END AS "Gold"
FROM weighted
FROM agg
CROSS JOIN totals
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
agg."InventoryId", agg."Name", agg."Vcenter", agg."VmId", agg."EventKey", agg."CloudId",
agg."Datacenter", agg."Cluster", agg."Folder", agg."ProvisionedDisk", agg."VcpuCount",
agg."RamGB", agg."IsTemplate", agg."PoweredOn", agg."SrmPlaceholder", agg."VmUuid",
agg.any_creation, agg.any_deletion, agg.first_present, agg.last_present, agg.last_absent,
totals.total_samples;
`, unionQuery, tableName)
return insert, nil
}
@@ -487,22 +494,18 @@ func BuildMonthlySummaryInsert(tableName string, unionQuery string) (string, err
return "", err
}
insert := fmt.Sprintf(`
WITH snapshots AS (
WITH daily AS (
%s
), ordered AS (
), enriched AS (
SELECT
s.*,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot,
LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds
FROM snapshots s
), weighted AS (
SELECT
o.*,
d.*,
CASE
WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600
ELSE o.interval_seconds
END AS weight_seconds
FROM ordered o
WHEN d."AvgIsPresent" IS NOT NULL AND d."AvgIsPresent" > 0 THEN d."SamplesPresent" / d."AvgIsPresent"
ELSE CAST(d."SamplesPresent" AS REAL)
END AS total_samples_day
FROM daily d
), totals AS (
SELECT COALESCE(SUM(total_samples_day), 0) AS total_samples FROM enriched
)
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
@@ -514,68 +517,50 @@ INSERT INTO %s (
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime",
COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime",
(
SELECT s2."ResourcePool"
FROM weighted s2
WHERE s2."VmId" = weighted."VmId"
AND s2."Vcenter" = weighted."Vcenter"
AND s2."IsPresent" = 'TRUE'
ORDER BY s2."SnapshotTime" DESC
LIMIT 1
) AS "ResourcePool",
COALESCE(NULLIF("CreationTime", 0), MIN(NULLIF("CreationTime", 0)), 0) AS "CreationTime",
NULLIF(MAX(NULLIF("DeletionTime", 0)), 0) AS "DeletionTime",
MAX("ResourcePool") AS "ResourcePool",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
SUM("SamplesPresent") AS "SamplesPresent",
CASE WHEN totals.total_samples > 0
THEN SUM(CASE WHEN "AvgVcpuCount" IS NOT NULL THEN "AvgVcpuCount" * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "AvgVcpuCount",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN SUM(CASE WHEN "AvgRamGB" IS NOT NULL THEN "AvgRamGB" * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "AvgRamGB",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN SUM(CASE WHEN "AvgProvisionedDisk" IS NOT NULL THEN "AvgProvisionedDisk" * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "AvgProvisionedDisk",
CASE WHEN SUM(weight_seconds) > 0
THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds)
CASE WHEN totals.total_samples > 0
THEN SUM("SamplesPresent") * 1.0 / totals.total_samples
ELSE NULL END AS "AvgIsPresent",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "PoolTinPct" IS NOT NULL THEN ("PoolTinPct" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "PoolTinPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "PoolBronzePct" IS NOT NULL THEN ("PoolBronzePct" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "PoolBronzePct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "PoolSilverPct" IS NOT NULL THEN ("PoolSilverPct" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "PoolSilverPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "PoolGoldPct" IS NOT NULL THEN ("PoolGoldPct" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "PoolGoldPct",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "Tin" IS NOT NULL THEN ("Tin" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "Tin",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "Bronze" IS NOT NULL THEN ("Bronze" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "Bronze",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "Silver" IS NOT NULL THEN ("Silver" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "Silver",
CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0
THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END)
/ SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END)
CASE WHEN totals.total_samples > 0
THEN 100.0 * SUM(CASE WHEN "Gold" IS NOT NULL THEN ("Gold" / 100.0) * total_samples_day ELSE 0 END) / totals.total_samples
ELSE NULL END AS "Gold"
FROM weighted
FROM enriched
CROSS JOIN totals
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",