From 29c277f86319438b435fa3bdc26c2e950829cc80 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Wed, 18 Feb 2026 11:59:22 +1100 Subject: [PATCH] Add support for customizable pivot titles and ranges in summary reports --- README.md | 6 + internal/report/snapshots.go | 187 +++++++++++++++------ internal/report/snapshots_pivot_test.go | 2 +- internal/settings/settings.go | 87 +++++----- internal/tasks/inventorySnapshots.go | 2 +- server/handler/snapshotRegenerateHourly.go | 2 +- server/handler/snapshots.go | 2 +- src/vctp.yml | 19 +++ 8 files changed, 217 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 420ce4c..ed7961b 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,12 @@ Snapshots: - `settings.hourly_index_max_age_days`: age gate for keeping per-hourly-table indexes (`-1` disables cleanup, `0` trims all) - `settings.snapshot_cleanup_cron`: cron expression for cleanup job - `settings.reports_dir`: directory to store generated XLSX reports (default: `/var/lib/vctp/reports`) +- `settings.report_summary_pivots`: optional list to override Summary worksheet pivot titles/names/ranges in daily/monthly XLSX reports + - `metric`: one of `avg_vcpu`, `avg_ram`, `prorated_vm_count`, `vm_name_count` + - `title`: pivot title text shown on Summary sheet + - `pivot_name`: internal pivot table name in the XLSX workbook + - `pivot_range`: target range (for example `Summary!A3:H40` or `A3:H40`) + - `title_cell` (optional): explicit title cell; if omitted, derived from `pivot_range` - `settings.hourly_snapshot_retry_seconds`: interval for retrying failed hourly snapshots (default: 300 seconds) - `settings.hourly_snapshot_max_retries`: maximum retry attempts per vCenter snapshot (default: 3) diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index 5e71923..6efe24b 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -15,6 +15,7 @@ import ( "sync" "time" "vctp/db" + "vctp/internal/settings" "github.com/jmoiron/sqlx" "github.com/xuri/excelize/v2" @@ -545,7 +546,7 @@ func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName } } -func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string) ([]byte, error) { +func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string, cfg *settings.Settings) ([]byte, error) { if err := db.ValidateTableName(tableName); err != nil { return nil, err } @@ -753,7 +754,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co for _, spec := range specs { reportHeaders = append(reportHeaders, spec.Name) } - addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName) + addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName, cfg) meta := reportMetadata{ TableName: tableName, @@ -787,7 +788,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co } // SaveTableReport renders a table report and writes it to the destination directory with a .xlsx extension. -func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string) (string, error) { +func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string, cfg *settings.Settings) (string, error) { if err := db.ValidateTableName(tableName); err != nil { return "", err } @@ -802,7 +803,7 @@ func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Cont } logger.Debug("Report directory ready", "dest", destDir) - data, err := CreateTableReport(logger, Database, ctx, tableName) + data, err := CreateTableReport(logger, Database, ctx, tableName, cfg) if err != nil { logger.Warn("Report render failed", "table", tableName, "error", err) return "", err @@ -890,6 +891,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context. } type summaryPivotSpec struct { + Metric string Title string TitleCell string PivotName string @@ -900,7 +902,139 @@ type summaryPivotSpec struct { DataSummary string } -func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string) { +func defaultSummaryPivotSpecs(summarySheet string) []summaryPivotSpec { + return []summaryPivotSpec{ + { + Metric: "avg_vcpu", + Title: "Sum of Avg vCPUs", + TitleCell: "A1", + PivotName: "PivotAvgVcpu", + PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet), + RowFields: []string{"Datacenter", "ResourcePool"}, + DataField: "AvgVcpuCount", + DataName: "Sum of Avg vCPUs", + DataSummary: "Sum", + }, + { + Metric: "avg_ram", + Title: "Sum of Avg RAM", + TitleCell: "J1", + PivotName: "PivotAvgRam", + PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet), + RowFields: []string{"Datacenter"}, + DataField: "AvgRamGB", + DataName: "Sum of Avg RAM", + DataSummary: "Sum", + }, + { + Metric: "prorated_vm_count", + Title: "Sum of prorated VM count", + TitleCell: "A61", + PivotName: "PivotProratedVmCount", + PivotRange: fmt.Sprintf("%s!A63:H82", summarySheet), + RowFields: []string{"Datacenter"}, + DataField: "AvgIsPresent", + DataName: "Sum of prorated VM count", + DataSummary: "Sum", + }, + { + Metric: "vm_name_count", + Title: "Count of VM Name", + TitleCell: "J61", + PivotName: "PivotVmNameCount", + PivotRange: fmt.Sprintf("%s!J63:P82", summarySheet), + RowFields: []string{"Datacenter"}, + DataField: "Name", + DataName: "Count of VM Name", + DataSummary: "Count", + }, + } +} + +func normalizePivotRange(summarySheet, pivotRange string) string { + trimmed := strings.TrimSpace(pivotRange) + if trimmed == "" { + return "" + } + if strings.Contains(trimmed, "!") { + return trimmed + } + return fmt.Sprintf("%s!%s", summarySheet, trimmed) +} + +func titleCellFromPivotRange(pivotRange, fallback string) string { + trimmed := strings.TrimSpace(pivotRange) + if trimmed == "" { + return fallback + } + if bang := strings.Index(trimmed, "!"); bang >= 0 { + trimmed = trimmed[bang+1:] + } + trimmed = strings.ReplaceAll(trimmed, "$", "") + startCell := trimmed + if parts := strings.Split(trimmed, ":"); len(parts) > 0 { + startCell = parts[0] + } + col, row, err := excelize.CellNameToCoordinates(startCell) + if err != nil { + return fallback + } + titleRow := row - 2 + if titleRow < 1 { + titleRow = 1 + } + cell, err := excelize.CoordinatesToCellName(col, titleRow) + if err != nil { + return fallback + } + return cell +} + +func resolveSummaryPivotSpecs(cfg *settings.Settings, summarySheet string) []summaryPivotSpec { + specs := defaultSummaryPivotSpecs(summarySheet) + if cfg == nil || cfg.Values == nil || len(cfg.Values.Settings.ReportSummaryPivots) == 0 { + return specs + } + + specByMetric := make(map[string]*summaryPivotSpec, len(specs)) + metricOrder := make([]string, 0, len(specs)) + for i := range specs { + metric := strings.TrimSpace(strings.ToLower(specs[i].Metric)) + if metric == "" { + continue + } + specByMetric[metric] = &specs[i] + metricOrder = append(metricOrder, metric) + } + + for i, custom := range cfg.Values.Settings.ReportSummaryPivots { + metric := strings.TrimSpace(strings.ToLower(custom.Metric)) + if metric == "" && i < len(metricOrder) { + metric = metricOrder[i] + } + spec := specByMetric[metric] + if spec == nil { + continue + } + if title := strings.TrimSpace(custom.Title); title != "" { + spec.Title = title + } + if name := strings.TrimSpace(custom.PivotName); name != "" { + spec.PivotName = name + } + if rng := normalizePivotRange(summarySheet, custom.PivotRange); rng != "" { + spec.PivotRange = rng + } + if cell := strings.TrimSpace(custom.TitleCell); cell != "" { + spec.TitleCell = cell + } else { + spec.TitleCell = titleCellFromPivotRange(spec.PivotRange, spec.TitleCell) + } + } + return specs +} + +func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string, cfg *settings.Settings) { if logger == nil { logger = slog.Default() } @@ -940,48 +1074,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st return header, ok } - specs := []summaryPivotSpec{ - { - Title: "Sum of Avg vCPUs", - TitleCell: "A1", - PivotName: "PivotAvgVcpu", - PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet), - RowFields: []string{"Datacenter", "ResourcePool"}, - DataField: "AvgVcpuCount", - DataName: "Sum of Avg vCPUs", - DataSummary: "Sum", - }, - { - Title: "Sum of Avg RAM", - TitleCell: "J1", - PivotName: "PivotAvgRam", - PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet), - RowFields: []string{"Datacenter"}, - DataField: "AvgRamGB", - DataName: "Sum of Avg RAM", - DataSummary: "Sum", - }, - { - Title: "Sum of prorated VM count", - TitleCell: "A41", - PivotName: "PivotProratedVmCount", - PivotRange: fmt.Sprintf("%s!A43:H62", summarySheet), - RowFields: []string{"Datacenter"}, - DataField: "AvgIsPresent", - DataName: "Sum of prorated VM count", - DataSummary: "Sum", - }, - { - Title: "Count of VM Name", - TitleCell: "J41", - PivotName: "PivotVmNameCount", - PivotRange: fmt.Sprintf("%s!J43:P62", summarySheet), - RowFields: []string{"Datacenter"}, - DataField: "Name", - DataName: "Count of VM Name", - DataSummary: "Count", - }, - } + specs := resolveSummaryPivotSpecs(cfg, summarySheet) for _, spec := range specs { xlsx.SetCellValue(summarySheet, spec.TitleCell, spec.Title) diff --git a/internal/report/snapshots_pivot_test.go b/internal/report/snapshots_pivot_test.go index d0641e8..4d9a234 100644 --- a/internal/report/snapshots_pivot_test.go +++ b/internal/report/snapshots_pivot_test.go @@ -27,7 +27,7 @@ func TestAddSummaryPivotSheetCreatesPivotTables(t *testing.T) { } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215") + addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215", nil) pivots, err := xlsx.GetPivotTables("Summary") if err != nil { diff --git a/internal/settings/settings.go b/internal/settings/settings.go index a0dc0cc..53ef1ef 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -24,48 +24,57 @@ type Settings struct { Values *SettingsYML } +type ReportSummaryPivot struct { + Metric string `yaml:"metric"` + Title string `yaml:"title"` + PivotName string `yaml:"pivot_name"` + PivotRange string `yaml:"pivot_range"` + TitleCell string `yaml:"title_cell"` +} + // SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties type SettingsYML struct { Settings struct { - LogLevel string `yaml:"log_level"` - LogOutput string `yaml:"log_output"` - DatabaseDriver string `yaml:"database_driver"` - DatabaseURL string `yaml:"database_url"` - EnableExperimentalPostgres bool `yaml:"enable_experimental_postgres"` - BindIP string `yaml:"bind_ip"` - BindPort int `yaml:"bind_port"` - BindDisableTLS bool `yaml:"bind_disable_tls"` - TLSCertFilename string `yaml:"tls_cert_filename"` - TLSKeyFilename string `yaml:"tls_key_filename"` - EncryptionKey string `yaml:"encryption_key"` - VcenterUsername string `yaml:"vcenter_username"` - VcenterPassword string `yaml:"vcenter_password"` - VcenterInsecure bool `yaml:"vcenter_insecure"` - EnableLegacyAPI bool `yaml:"enable_legacy_api"` - VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"` - VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"` - VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"` - VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"` - HourlySnapshotConcurrency int `yaml:"hourly_snapshot_concurrency"` - HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"` - DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"` - HourlyIndexMaxAgeDays int `yaml:"hourly_index_max_age_days"` - SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"` - ReportsDir string `yaml:"reports_dir"` - HourlyJobTimeoutSeconds int `yaml:"hourly_job_timeout_seconds"` - HourlySnapshotTimeoutSeconds int `yaml:"hourly_snapshot_timeout_seconds"` - HourlySnapshotRetrySeconds int `yaml:"hourly_snapshot_retry_seconds"` - HourlySnapshotMaxRetries int `yaml:"hourly_snapshot_max_retries"` - DailyJobTimeoutSeconds int `yaml:"daily_job_timeout_seconds"` - MonthlyJobTimeoutSeconds int `yaml:"monthly_job_timeout_seconds"` - MonthlyAggregationGranularity string `yaml:"monthly_aggregation_granularity"` - MonthlyAggregationCron string `yaml:"monthly_aggregation_cron"` - CleanupJobTimeoutSeconds int `yaml:"cleanup_job_timeout_seconds"` - TenantsToFilter []string `yaml:"tenants_to_filter"` - NodeChargeClusters []string `yaml:"node_charge_clusters"` - SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"` - VcenterAddresses []string `yaml:"vcenter_addresses"` - PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"` + LogLevel string `yaml:"log_level"` + LogOutput string `yaml:"log_output"` + DatabaseDriver string `yaml:"database_driver"` + DatabaseURL string `yaml:"database_url"` + EnableExperimentalPostgres bool `yaml:"enable_experimental_postgres"` + BindIP string `yaml:"bind_ip"` + BindPort int `yaml:"bind_port"` + BindDisableTLS bool `yaml:"bind_disable_tls"` + TLSCertFilename string `yaml:"tls_cert_filename"` + TLSKeyFilename string `yaml:"tls_key_filename"` + EncryptionKey string `yaml:"encryption_key"` + VcenterUsername string `yaml:"vcenter_username"` + VcenterPassword string `yaml:"vcenter_password"` + VcenterInsecure bool `yaml:"vcenter_insecure"` + EnableLegacyAPI bool `yaml:"enable_legacy_api"` + VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"` + VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"` + VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"` + VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"` + HourlySnapshotConcurrency int `yaml:"hourly_snapshot_concurrency"` + HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"` + DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"` + HourlyIndexMaxAgeDays int `yaml:"hourly_index_max_age_days"` + SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"` + ReportsDir string `yaml:"reports_dir"` + HourlyJobTimeoutSeconds int `yaml:"hourly_job_timeout_seconds"` + HourlySnapshotTimeoutSeconds int `yaml:"hourly_snapshot_timeout_seconds"` + HourlySnapshotRetrySeconds int `yaml:"hourly_snapshot_retry_seconds"` + HourlySnapshotMaxRetries int `yaml:"hourly_snapshot_max_retries"` + DailyJobTimeoutSeconds int `yaml:"daily_job_timeout_seconds"` + MonthlyJobTimeoutSeconds int `yaml:"monthly_job_timeout_seconds"` + MonthlyAggregationGranularity string `yaml:"monthly_aggregation_granularity"` + MonthlyAggregationCron string `yaml:"monthly_aggregation_cron"` + CleanupJobTimeoutSeconds int `yaml:"cleanup_job_timeout_seconds"` + TenantsToFilter []string `yaml:"tenants_to_filter"` + NodeChargeClusters []string `yaml:"node_charge_clusters"` + SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"` + VcenterAddresses []string `yaml:"vcenter_addresses"` + PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"` + ReportSummaryPivots []ReportSummaryPivot `yaml:"report_summary_pivots"` } `yaml:"settings"` } diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index 7df9a1d..62a0b46 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -676,7 +676,7 @@ func (c *CronTask) generateReport(ctx context.Context, tableName string) error { dest := c.reportsDir() start := time.Now() c.Logger.Debug("Report generation start", "table", tableName, "dest", dest) - filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest) + filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest, c.Settings) if err == nil { c.Logger.Debug("Report generation complete", "table", tableName, "file", filename, "duration", time.Since(start)) } else { diff --git a/server/handler/snapshotRegenerateHourly.go b/server/handler/snapshotRegenerateHourly.go index 6245c52..726624a 100644 --- a/server/handler/snapshotRegenerateHourly.go +++ b/server/handler/snapshotRegenerateHourly.go @@ -46,7 +46,7 @@ func (h *Handler) SnapshotRegenerateHourlyReports(w http.ResponseWriter, r *http skipped++ continue } - if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir); err != nil { + if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir, h.Settings); err != nil { errors++ h.Logger.Warn("failed to regenerate hourly report", "table", rec.TableName, "error", err) continue diff --git a/server/handler/snapshots.go b/server/handler/snapshots.go index 7324288..115b5b6 100644 --- a/server/handler/snapshots.go +++ b/server/handler/snapshots.go @@ -65,7 +65,7 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request) return } - reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName) + reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName, h.Settings) if err != nil { h.Logger.Error("Failed to create snapshot report", "error", err, "table", tableName) writeJSONError(w, http.StatusInternalServerError, fmt.Sprintf("Unable to create snapshot report: '%s'", err)) diff --git a/src/vctp.yml b/src/vctp.yml index 6b7f27b..130e02a 100644 --- a/src/vctp.yml +++ b/src/vctp.yml @@ -40,6 +40,25 @@ settings: monthly_job_timeout_seconds: 1200 monthly_aggregation_granularity: "hourly" monthly_aggregation_cron: "10 3 1 * *" + # Optional: override Summary worksheet pivot layout in daily/monthly XLSX reports. + # metric values: avg_vcpu, avg_ram, prorated_vm_count, vm_name_count + report_summary_pivots: + - metric: avg_vcpu + title: "Sum of Avg vCPUs" + pivot_name: "PivotAvgVcpu" + pivot_range: "Summary!A3:H40" + - metric: avg_ram + title: "Sum of Avg RAM" + pivot_name: "PivotAvgRam" + pivot_range: "Summary!J3:P40" + - metric: prorated_vm_count + title: "Sum of prorated VM count" + pivot_name: "PivotProratedVmCount" + pivot_range: "Summary!A63:H82" + - metric: vm_name_count + title: "Count of VM Name" + pivot_name: "PivotVmNameCount" + pivot_range: "Summary!J63:P82" cleanup_job_timeout_seconds: 600 tenants_to_filter: node_charge_clusters: