nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress on go based aggregation
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-16 17:37:55 +11:00
parent 6af49471b2
commit 1cd1046433
5 changed files with 758 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
db/helpers.go
View File

@@ -394,6 +394,115 @@ func ApplySQLiteTuning(ctx context.Context, dbConn *sqlx.DB) {
}
}
// EnsureVmHourlyStats creates the shared per-snapshot cache table used by Go aggregations.
func EnsureVmHourlyStats(ctx context.Context, dbConn *sqlx.DB) error {
ddl := `
CREATE TABLE IF NOT EXISTS vm_hourly_stats (
"SnapshotTime" BIGINT NOT NULL,
"Vcenter" TEXT NOT NULL,
"VmId" TEXT,
"VmUuid" TEXT,
"Name" TEXT,
"CreationTime" BIGINT,
"DeletionTime" BIGINT,
"ResourcePool" TEXT,
"Datacenter" TEXT,
"Cluster" TEXT,
"Folder" TEXT,
"ProvisionedDisk" REAL,
"VcpuCount" BIGINT,
"RamGB" BIGINT,
"IsTemplate" TEXT,
"PoweredOn" TEXT,
"SrmPlaceholder" TEXT,
PRIMARY KEY ("Vcenter","VmId","SnapshotTime")
);`
if _, err := execLog(ctx, dbConn, ddl); err != nil {
return err
}
_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_hourly_stats_vmuuid_time_idx ON vm_hourly_stats ("VmUuid","SnapshotTime")`)
_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_hourly_stats_snapshottime_idx ON vm_hourly_stats ("SnapshotTime")`)
return nil
}
// EnsureVmLifecycleCache creates an upsert cache for first/last seen VM info.
func EnsureVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB) error {
ddl := `
CREATE TABLE IF NOT EXISTS vm_lifecycle_cache (
"Vcenter" TEXT NOT NULL,
"VmId" TEXT,
"VmUuid" TEXT,
"Name" TEXT,
"Cluster" TEXT,
"FirstSeen" BIGINT,
"LastSeen" BIGINT,
"DeletedAt" BIGINT,
PRIMARY KEY ("Vcenter","VmId","VmUuid")
);`
if _, err := execLog(ctx, dbConn, ddl); err != nil {
return err
}
_, _ = execLog(ctx, dbConn, `CREATE INDEX IF NOT EXISTS vm_lifecycle_cache_vmuuid_idx ON vm_lifecycle_cache ("VmUuid")`)
return nil
}
// UpsertVmLifecycleCache updates first/last seen info for a VM.
func UpsertVmLifecycleCache(ctx context.Context, dbConn *sqlx.DB, vcenter string, vmID, vmUUID, name, cluster string, seen time.Time) error {
if err := EnsureVmLifecycleCache(ctx, dbConn); err != nil {
return err
}
driver := strings.ToLower(dbConn.DriverName())
query := `
INSERT INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","Name","Cluster","FirstSeen","LastSeen")
VALUES ($1,$2,$3,$4,$5,$6,$6)
ON CONFLICT ("Vcenter","VmId","VmUuid") DO UPDATE SET
"Name"=EXCLUDED."Name",
"Cluster"=EXCLUDED."Cluster",
"LastSeen"=EXCLUDED."LastSeen",
"FirstSeen"=COALESCE(vm_lifecycle_cache."FirstSeen", EXCLUDED."FirstSeen"),
"DeletedAt"=NULL
`
args := []interface{}{vcenter, vmID, vmUUID, name, cluster, seen.Unix()}
if driver == "sqlite" {
query = `
INSERT OR REPLACE INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","Name","Cluster","FirstSeen","LastSeen")
VALUES (?,?,?,?,?,?,?)
`
}
_, err := dbConn.ExecContext(ctx, query, args...)
return err
}
// MarkVmDeleted updates lifecycle cache with a deletion timestamp.
func MarkVmDeleted(ctx context.Context, dbConn *sqlx.DB, vcenter, vmID, vmUUID string, deletedAt int64) error {
if err := EnsureVmLifecycleCache(ctx, dbConn); err != nil {
return err
}
driver := strings.ToLower(dbConn.DriverName())
query := `
INSERT INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","DeletedAt","FirstSeen","LastSeen")
VALUES ($1,$2,$3,$4,$4,$4)
ON CONFLICT ("Vcenter","VmId","VmUuid") DO UPDATE SET
"DeletedAt"=CASE
WHEN vm_lifecycle_cache."DeletedAt" IS NULL OR vm_lifecycle_cache."DeletedAt"=0 OR EXCLUDED."DeletedAt"<vm_lifecycle_cache."DeletedAt"
THEN EXCLUDED."DeletedAt"
ELSE vm_lifecycle_cache."DeletedAt"
END,
"LastSeen"=COALESCE(vm_lifecycle_cache."LastSeen", EXCLUDED."LastSeen"),
"FirstSeen"=COALESCE(vm_lifecycle_cache."FirstSeen", EXCLUDED."FirstSeen")
`
args := []interface{}{vcenter, vmID, vmUUID, deletedAt}
if driver == "sqlite" {
query = `
INSERT OR REPLACE INTO vm_lifecycle_cache ("Vcenter","VmId","VmUuid","DeletedAt","FirstSeen","LastSeen")
VALUES (?,?,?,?,?,?)
`
args = []interface{}{vcenter, vmID, vmUUID, deletedAt, deletedAt, deletedAt}
}
_, err := dbConn.ExecContext(ctx, query, args...)
return err
}
// EnsureVmIdentityTables creates the identity and rename audit tables.
func EnsureVmIdentityTables(ctx context.Context, dbConn *sqlx.DB) error {
driver := strings.ToLower(dbConn.DriverName())
@@ -1318,8 +1427,13 @@ SELECT
COALESCE(NULLIF("CreationTime", 0), MIN(NULLIF("CreationTime", 0)), 0) AS "CreationTime",
NULLIF(MAX(NULLIF("DeletionTime", 0)), 0) AS "DeletionTime",
MAX("ResourcePool") AS "ResourcePool",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"Datacenter", "Cluster", "Folder",
MAX("ProvisionedDisk") AS "ProvisionedDisk",
MAX("VcpuCount") AS "VcpuCount",
MAX("RamGB") AS "RamGB",
"IsTemplate",
MAX("PoweredOn") AS "PoweredOn",
"SrmPlaceholder", "VmUuid",
SUM("SamplesPresent") AS "SamplesPresent",
CASE WHEN totals.total_samples > 0
THEN SUM(CASE WHEN "AvgVcpuCount" IS NOT NULL THEN "AvgVcpuCount" * total_samples_day ELSE 0 END) / totals.total_samples
@@ -1361,8 +1475,8 @@ FROM enriched
CROSS JOIN totals
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId",
"Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
"Datacenter", "Cluster", "Folder",
"IsTemplate", "SrmPlaceholder", "VmUuid";
`, unionQuery, tableName)
return insert, nil
}