diff --git a/.gitignore b/.gitignore index 7e09d43..0f7e2c6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ vctp build/ reports/ +reports/*.xlsx settings.yaml # Certificates @@ -44,7 +45,7 @@ appengine-generated/ tmp/ pb_data/ -# General +# Generalis .DS_Store .AppleDouble .LSOverride diff --git a/db/helpers.go b/db/helpers.go index 76599f7..a8dfcab 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -281,9 +281,17 @@ func EnsureSnapshotTable(ctx context.Context, dbConn *sqlx.DB, tableName string) return err } - index := fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName) - _, err = dbConn.ExecContext(ctx, index) - return err + indexes := []string{ + fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName), + fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_snapshottime_idx ON %s ("SnapshotTime")`, tableName, tableName), + fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName), + } + for _, idx := range indexes { + if _, err := dbConn.ExecContext(ctx, idx); err != nil { + return err + } + } + return nil } // BackfillSerialColumn sets missing values in a serial-like column for Postgres tables. @@ -378,6 +386,20 @@ func BuildDailySummaryInsert(tableName string, unionQuery string) (string, error insert := fmt.Sprintf(` WITH snapshots AS ( %s +), ordered AS ( + SELECT + s.*, + LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot, + LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds + FROM snapshots s +), weighted AS ( + SELECT + o.*, + CASE + WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600 + ELSE o.interval_seconds + END AS weight_seconds + FROM ordered o ) INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", @@ -390,12 +412,12 @@ INSERT INTO %s ( SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", - "DeletionTime", + COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime", ( SELECT s2."ResourcePool" - FROM snapshots s2 - WHERE s2."VmId" = snapshots."VmId" - AND s2."Vcenter" = snapshots."Vcenter" + FROM weighted s2 + WHERE s2."VmId" = weighted."VmId" + AND s2."Vcenter" = weighted."Vcenter" AND s2."IsPresent" = 'TRUE' ORDER BY s2."SnapshotTime" DESC LIMIT 1 @@ -403,29 +425,56 @@ SELECT "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", - AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" -FROM snapshots + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgVcpuCount", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgRamGB", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgProvisionedDisk", + CASE WHEN SUM(weight_seconds) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds) + ELSE NULL END AS "AvgIsPresent", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolTinPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolBronzePct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolSilverPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolGoldPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Tin", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Bronze", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Silver", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Gold" +FROM weighted GROUP BY - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, unionQuery, tableName) @@ -440,51 +489,95 @@ func BuildMonthlySummaryInsert(tableName string, unionQuery string) (string, err insert := fmt.Sprintf(` WITH snapshots AS ( %s +), ordered AS ( + SELECT + s.*, + LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") AS next_snapshot, + LEAD("SnapshotTime") OVER (PARTITION BY "VmId", "Vcenter" ORDER BY "SnapshotTime") - "SnapshotTime" AS interval_seconds + FROM snapshots s +), weighted AS ( + SELECT + o.*, + CASE + WHEN o.interval_seconds IS NULL OR o.interval_seconds <= 0 THEN 3600 + ELSE o.interval_seconds + END AS weight_seconds + FROM ordered o ) INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", - "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct", "Tin", "Bronze", "Silver", "Gold" ) SELECT - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", + COALESCE(NULLIF("CreationTime", 0), MIN(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "CreationTime", + COALESCE(NULLIF("DeletionTime", 0), MAX(CASE WHEN "IsPresent" = 'TRUE' THEN "SnapshotTime" END), 0) AS "DeletionTime", ( SELECT s2."ResourcePool" - FROM snapshots s2 - WHERE s2."VmId" = snapshots."VmId" - AND s2."Vcenter" = snapshots."Vcenter" + FROM weighted s2 + WHERE s2."VmId" = weighted."VmId" + AND s2."Vcenter" = weighted."Vcenter" AND s2."IsPresent" = 'TRUE' ORDER BY s2."SnapshotTime" DESC LIMIT 1 ) AS "ResourcePool", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", - AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", - AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", - AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolBronzePct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolSilverPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolGoldPct", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Tin", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Bronze", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Silver", - 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN 1 ELSE 0 END) - / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "Gold" -FROM snapshots + SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgVcpuCount", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgRamGB", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" * weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "AvgProvisionedDisk", + CASE WHEN SUM(weight_seconds) > 0 + THEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) / SUM(weight_seconds) + ELSE NULL END AS "AvgIsPresent", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolTinPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolBronzePct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolSilverPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "PoolGoldPct", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Tin", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'bronze' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Bronze", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'silver' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Silver", + CASE WHEN SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) > 0 + THEN 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'gold' THEN weight_seconds ELSE 0 END) + / SUM(CASE WHEN "IsPresent" = 'TRUE' THEN weight_seconds ELSE 0 END) + ELSE NULL END AS "Gold" +FROM weighted GROUP BY - "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", + "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, unionQuery, tableName) @@ -574,6 +667,18 @@ func EnsureSummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) );`, tableName) } - _, err := dbConn.ExecContext(ctx, ddl) - return err + if _, err := dbConn.ExecContext(ctx, ddl); err != nil { + return err + } + + indexes := []string{ + fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName), + fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName), + } + for _, idx := range indexes { + if _, err := dbConn.ExecContext(ctx, idx); err != nil { + return err + } + } + return nil } diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index b3c632b..ff23ad4 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "path/filepath" + "sort" "strconv" "strings" "time" @@ -341,6 +342,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp startUnix := start.Unix() endUnix := end.Unix() + loc := start.Location() var rows *sqlx.Rows var err error @@ -386,7 +388,7 @@ ORDER BY snapshot_time ASC, table_name ASC } records = append(records, SnapshotRecord{ TableName: tableName, - SnapshotTime: time.Unix(snapshotTime, 0), + SnapshotTime: time.Unix(snapshotTime, 0).In(loc), SnapshotType: recordType, SnapshotCount: snapshotCnt, }) @@ -394,6 +396,65 @@ ORDER BY snapshot_time ASC, table_name ASC return records, rows.Err() } +func SnapshotRecordsWithFallback(ctx context.Context, database db.Database, snapshotType, prefix, layout string, start, end time.Time) ([]SnapshotRecord, error) { + records, err := ListSnapshotsByRange(ctx, database, snapshotType, start, end) + if err == nil && len(records) > 0 { + return records, nil + } + + fallback, err2 := recordsFromTableNames(ctx, database, snapshotType, prefix, layout, start, end) + if err2 != nil { + if err != nil { + return nil, err + } + return nil, err2 + } + if len(fallback) > 0 { + return fallback, nil + } + return records, err +} + +func recordsFromTableNames(ctx context.Context, database db.Database, snapshotType, prefix, layout string, start, end time.Time) ([]SnapshotRecord, error) { + tables, err := ListTablesByPrefix(ctx, database, prefix) + if err != nil { + return nil, err + } + records := make([]SnapshotRecord, 0, len(tables)) + for _, table := range tables { + if !strings.HasPrefix(table, prefix) { + continue + } + suffix := strings.TrimPrefix(table, prefix) + var ts time.Time + switch layout { + case "epoch": + val, err := strconv.ParseInt(suffix, 10, 64) + if err != nil { + continue + } + ts = time.Unix(val, 0) + default: + parsed, err := time.ParseInLocation(layout, suffix, time.Local) + if err != nil { + continue + } + ts = parsed + } + if !ts.Before(start) && ts.Before(end) { + records = append(records, SnapshotRecord{ + TableName: table, + SnapshotTime: ts, + SnapshotType: snapshotType, + }) + } + } + sort.Slice(records, func(i, j int) bool { + return records[i].SnapshotTime.Before(records[j].SnapshotTime) + }) + return records, nil +} + func LatestSnapshotTime(ctx context.Context, database db.Database, snapshotType string) (time.Time, error) { dbConn := database.DB() driver := strings.ToLower(dbConn.DriverName()) @@ -665,7 +726,7 @@ func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Cont func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.Context, xlsx *excelize.File, tableName string) { if strings.HasPrefix(tableName, "inventory_daily_summary_") { suffix := strings.TrimPrefix(tableName, "inventory_daily_summary_") - dayStart, err := time.Parse("20060102", suffix) + dayStart, err := time.ParseInLocation("20060102", suffix, time.Local) if err != nil { return } @@ -673,7 +734,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context. if err := EnsureSnapshotRegistry(ctx, database); err != nil { return } - records, err := ListSnapshotsByRange(ctx, database, "hourly", dayStart, dayEnd) + records, err := SnapshotRecordsWithFallback(ctx, database, "hourly", "inventory_hourly_", "epoch", dayStart, dayEnd) if err != nil || len(records) == 0 { return } @@ -687,7 +748,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context. if strings.HasPrefix(tableName, "inventory_monthly_summary_") { suffix := strings.TrimPrefix(tableName, "inventory_monthly_summary_") - monthStart, err := time.Parse("200601", suffix) + monthStart, err := time.ParseInLocation("200601", suffix, time.Local) if err != nil { return } @@ -695,7 +756,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context. if err := EnsureSnapshotRegistry(ctx, database); err != nil { return } - records, err := ListSnapshotsByRange(ctx, database, "daily", monthStart, monthEnd) + records, err := SnapshotRecordsWithFallback(ctx, database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd) if err != nil || len(records) == 0 { return } diff --git a/internal/tasks/dailyAggregate.go b/internal/tasks/dailyAggregate.go index 01e84e2..d05fbe6 100644 --- a/internal/tasks/dailyAggregate.go +++ b/internal/tasks/dailyAggregate.go @@ -52,10 +52,11 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti } } - hourlySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", dayStart, dayEnd) + hourlySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", dayStart, dayEnd) if err != nil { return err } + hourlySnapshots = filterRecordsInRange(hourlySnapshots, dayStart, dayEnd) hourlySnapshots = filterSnapshotsWithRows(ctx, dbConn, hourlySnapshots) if len(hourlySnapshots) == 0 { return fmt.Errorf("no hourly snapshot tables found for %s", dayStart.Format("2006-01-02")) @@ -85,8 +86,9 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti prevStart := dayStart.AddDate(0, 0, -1) prevEnd := dayStart - prevSnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", prevStart, prevEnd) + prevSnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", prevStart, prevEnd) if err == nil && len(prevSnapshots) > 0 { + prevSnapshots = filterRecordsInRange(prevSnapshots, prevStart, prevEnd) prevSnapshots = filterSnapshotsWithRows(ctx, dbConn, prevSnapshots) prevTables := make([]string, 0, len(prevSnapshots)) for _, snapshot := range prevSnapshots { diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index 1b566e2..fd17de8 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -370,6 +370,16 @@ func filterSnapshotsWithRows(ctx context.Context, dbConn *sqlx.DB, snapshots []r return filtered } +func filterRecordsInRange(records []report.SnapshotRecord, start, end time.Time) []report.SnapshotRecord { + filtered := records[:0] + for _, r := range records { + if !r.SnapshotTime.Before(start) && r.SnapshotTime.Before(end) { + filtered = append(filtered, r) + } + } + return filtered +} + type columnDef struct { Name string Type string @@ -437,18 +447,17 @@ func normalizeResourcePool(value string) string { if trimmed == "" { return "" } - switch { - case strings.EqualFold(trimmed, "tin"): - return "Tin" - case strings.EqualFold(trimmed, "bronze"): - return "Bronze" - case strings.EqualFold(trimmed, "silver"): - return "Silver" - case strings.EqualFold(trimmed, "gold"): - return "Gold" - default: - return trimmed + lower := strings.ToLower(trimmed) + canonical := map[string]string{ + "tin": "Tin", + "bronze": "Bronze", + "silver": "Silver", + "gold": "Gold", } + if val, ok := canonical[lower]; ok { + return val + } + return trimmed } func (c *CronTask) reportsDir() string { diff --git a/internal/tasks/monthlyAggregate.go b/internal/tasks/monthlyAggregate.go index 02b6f13..e92ee85 100644 --- a/internal/tasks/monthlyAggregate.go +++ b/internal/tasks/monthlyAggregate.go @@ -35,10 +35,11 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time monthStart := time.Date(targetMonth.Year(), targetMonth.Month(), 1, 0, 0, 0, 0, targetMonth.Location()) monthEnd := monthStart.AddDate(0, 1, 0) - dailySnapshots, err := report.ListSnapshotsByRange(ctx, c.Database, "hourly", monthStart, monthEnd) + dailySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd) if err != nil { return err } + dailySnapshots = filterRecordsInRange(dailySnapshots, monthStart, monthEnd) dbConn := c.Database.DB() dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots) diff --git a/server/handler/snapshotAggregate.go b/server/handler/snapshotAggregate.go index 2a54fa9..682d0ab 100644 --- a/server/handler/snapshotAggregate.go +++ b/server/handler/snapshotAggregate.go @@ -24,6 +24,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request) snapshotType := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("type"))) dateValue := strings.TrimSpace(r.URL.Query().Get("date")) startedAt := time.Now() + loc := time.Now().Location() if snapshotType == "" || dateValue == "" { h.Logger.Warn("Snapshot aggregation request missing parameters", @@ -43,7 +44,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request) switch snapshotType { case "daily": - parsed, err := time.Parse("2006-01-02", dateValue) + parsed, err := time.ParseInLocation("2006-01-02", dateValue, loc) if err != nil { h.Logger.Warn("Snapshot aggregation invalid daily date format", "date", dateValue) writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM-DD") @@ -56,7 +57,7 @@ func (h *Handler) SnapshotAggregateForce(w http.ResponseWriter, r *http.Request) return } case "monthly": - parsed, err := time.Parse("2006-01", dateValue) + parsed, err := time.ParseInLocation("2006-01", dateValue, loc) if err != nil { h.Logger.Warn("Snapshot aggregation invalid monthly date format", "date", dateValue) writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM") diff --git a/src/vctp.service b/src/vctp.service index 629ce0c..d3fc691 100644 --- a/src/vctp.service +++ b/src/vctp.service @@ -1,5 +1,5 @@ [Unit] -Description=vCTP monitors VMware VM inventory and event data to build chargeback reports +Description=vSphere Chargeback Tracking Platform Documentation=https://gitlab.dell.com/ ConditionPathExists=/usr/bin/vctp-linux-amd64 After=network.target