Add support for customizable pivot titles and ranges in summary reports
Some checks failed
continuous-integration/drone/push Build was killed

This commit is contained in:
2026-02-18 11:59:22 +11:00
parent 92c6797f05
commit 29c277f863
8 changed files with 217 additions and 90 deletions

View File

@@ -279,6 +279,12 @@ Snapshots:
- `settings.hourly_index_max_age_days`: age gate for keeping per-hourly-table indexes (`-1` disables cleanup, `0` trims all)
- `settings.snapshot_cleanup_cron`: cron expression for cleanup job
- `settings.reports_dir`: directory to store generated XLSX reports (default: `/var/lib/vctp/reports`)
- `settings.report_summary_pivots`: optional list to override Summary worksheet pivot titles/names/ranges in daily/monthly XLSX reports
- `metric`: one of `avg_vcpu`, `avg_ram`, `prorated_vm_count`, `vm_name_count`
- `title`: pivot title text shown on Summary sheet
- `pivot_name`: internal pivot table name in the XLSX workbook
- `pivot_range`: target range (for example `Summary!A3:H40` or `A3:H40`)
- `title_cell` (optional): explicit title cell; if omitted, derived from `pivot_range`
- `settings.hourly_snapshot_retry_seconds`: interval for retrying failed hourly snapshots (default: 300 seconds)
- `settings.hourly_snapshot_max_retries`: maximum retry attempts per vCenter snapshot (default: 3)

View File

@@ -15,6 +15,7 @@ import (
"sync"
"time"
"vctp/db"
"vctp/internal/settings"
"github.com/jmoiron/sqlx"
"github.com/xuri/excelize/v2"
@@ -545,7 +546,7 @@ func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName
}
}
func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string) ([]byte, error) {
func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string, cfg *settings.Settings) ([]byte, error) {
if err := db.ValidateTableName(tableName); err != nil {
return nil, err
}
@@ -753,7 +754,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
for _, spec := range specs {
reportHeaders = append(reportHeaders, spec.Name)
}
addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName)
addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName, cfg)
meta := reportMetadata{
TableName: tableName,
@@ -787,7 +788,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
}
// SaveTableReport renders a table report and writes it to the destination directory with a .xlsx extension.
func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string) (string, error) {
func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string, cfg *settings.Settings) (string, error) {
if err := db.ValidateTableName(tableName); err != nil {
return "", err
}
@@ -802,7 +803,7 @@ func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Cont
}
logger.Debug("Report directory ready", "dest", destDir)
data, err := CreateTableReport(logger, Database, ctx, tableName)
data, err := CreateTableReport(logger, Database, ctx, tableName, cfg)
if err != nil {
logger.Warn("Report render failed", "table", tableName, "error", err)
return "", err
@@ -890,6 +891,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
}
type summaryPivotSpec struct {
Metric string
Title string
TitleCell string
PivotName string
@@ -900,7 +902,139 @@ type summaryPivotSpec struct {
DataSummary string
}
func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string) {
func defaultSummaryPivotSpecs(summarySheet string) []summaryPivotSpec {
return []summaryPivotSpec{
{
Metric: "avg_vcpu",
Title: "Sum of Avg vCPUs",
TitleCell: "A1",
PivotName: "PivotAvgVcpu",
PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet),
RowFields: []string{"Datacenter", "ResourcePool"},
DataField: "AvgVcpuCount",
DataName: "Sum of Avg vCPUs",
DataSummary: "Sum",
},
{
Metric: "avg_ram",
Title: "Sum of Avg RAM",
TitleCell: "J1",
PivotName: "PivotAvgRam",
PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgRamGB",
DataName: "Sum of Avg RAM",
DataSummary: "Sum",
},
{
Metric: "prorated_vm_count",
Title: "Sum of prorated VM count",
TitleCell: "A61",
PivotName: "PivotProratedVmCount",
PivotRange: fmt.Sprintf("%s!A63:H82", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgIsPresent",
DataName: "Sum of prorated VM count",
DataSummary: "Sum",
},
{
Metric: "vm_name_count",
Title: "Count of VM Name",
TitleCell: "J61",
PivotName: "PivotVmNameCount",
PivotRange: fmt.Sprintf("%s!J63:P82", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "Name",
DataName: "Count of VM Name",
DataSummary: "Count",
},
}
}
func normalizePivotRange(summarySheet, pivotRange string) string {
trimmed := strings.TrimSpace(pivotRange)
if trimmed == "" {
return ""
}
if strings.Contains(trimmed, "!") {
return trimmed
}
return fmt.Sprintf("%s!%s", summarySheet, trimmed)
}
func titleCellFromPivotRange(pivotRange, fallback string) string {
trimmed := strings.TrimSpace(pivotRange)
if trimmed == "" {
return fallback
}
if bang := strings.Index(trimmed, "!"); bang >= 0 {
trimmed = trimmed[bang+1:]
}
trimmed = strings.ReplaceAll(trimmed, "$", "")
startCell := trimmed
if parts := strings.Split(trimmed, ":"); len(parts) > 0 {
startCell = parts[0]
}
col, row, err := excelize.CellNameToCoordinates(startCell)
if err != nil {
return fallback
}
titleRow := row - 2
if titleRow < 1 {
titleRow = 1
}
cell, err := excelize.CoordinatesToCellName(col, titleRow)
if err != nil {
return fallback
}
return cell
}
func resolveSummaryPivotSpecs(cfg *settings.Settings, summarySheet string) []summaryPivotSpec {
specs := defaultSummaryPivotSpecs(summarySheet)
if cfg == nil || cfg.Values == nil || len(cfg.Values.Settings.ReportSummaryPivots) == 0 {
return specs
}
specByMetric := make(map[string]*summaryPivotSpec, len(specs))
metricOrder := make([]string, 0, len(specs))
for i := range specs {
metric := strings.TrimSpace(strings.ToLower(specs[i].Metric))
if metric == "" {
continue
}
specByMetric[metric] = &specs[i]
metricOrder = append(metricOrder, metric)
}
for i, custom := range cfg.Values.Settings.ReportSummaryPivots {
metric := strings.TrimSpace(strings.ToLower(custom.Metric))
if metric == "" && i < len(metricOrder) {
metric = metricOrder[i]
}
spec := specByMetric[metric]
if spec == nil {
continue
}
if title := strings.TrimSpace(custom.Title); title != "" {
spec.Title = title
}
if name := strings.TrimSpace(custom.PivotName); name != "" {
spec.PivotName = name
}
if rng := normalizePivotRange(summarySheet, custom.PivotRange); rng != "" {
spec.PivotRange = rng
}
if cell := strings.TrimSpace(custom.TitleCell); cell != "" {
spec.TitleCell = cell
} else {
spec.TitleCell = titleCellFromPivotRange(spec.PivotRange, spec.TitleCell)
}
}
return specs
}
func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string, cfg *settings.Settings) {
if logger == nil {
logger = slog.Default()
}
@@ -940,48 +1074,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
return header, ok
}
specs := []summaryPivotSpec{
{
Title: "Sum of Avg vCPUs",
TitleCell: "A1",
PivotName: "PivotAvgVcpu",
PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet),
RowFields: []string{"Datacenter", "ResourcePool"},
DataField: "AvgVcpuCount",
DataName: "Sum of Avg vCPUs",
DataSummary: "Sum",
},
{
Title: "Sum of Avg RAM",
TitleCell: "J1",
PivotName: "PivotAvgRam",
PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgRamGB",
DataName: "Sum of Avg RAM",
DataSummary: "Sum",
},
{
Title: "Sum of prorated VM count",
TitleCell: "A41",
PivotName: "PivotProratedVmCount",
PivotRange: fmt.Sprintf("%s!A43:H62", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgIsPresent",
DataName: "Sum of prorated VM count",
DataSummary: "Sum",
},
{
Title: "Count of VM Name",
TitleCell: "J41",
PivotName: "PivotVmNameCount",
PivotRange: fmt.Sprintf("%s!J43:P62", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "Name",
DataName: "Count of VM Name",
DataSummary: "Count",
},
}
specs := resolveSummaryPivotSpecs(cfg, summarySheet)
for _, spec := range specs {
xlsx.SetCellValue(summarySheet, spec.TitleCell, spec.Title)

View File

@@ -27,7 +27,7 @@ func TestAddSummaryPivotSheetCreatesPivotTables(t *testing.T) {
}
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215")
addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215", nil)
pivots, err := xlsx.GetPivotTables("Summary")
if err != nil {

View File

@@ -24,6 +24,14 @@ type Settings struct {
Values *SettingsYML
}
type ReportSummaryPivot struct {
Metric string `yaml:"metric"`
Title string `yaml:"title"`
PivotName string `yaml:"pivot_name"`
PivotRange string `yaml:"pivot_range"`
TitleCell string `yaml:"title_cell"`
}
// SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties
type SettingsYML struct {
Settings struct {
@@ -66,6 +74,7 @@ type SettingsYML struct {
SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"`
VcenterAddresses []string `yaml:"vcenter_addresses"`
PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"`
ReportSummaryPivots []ReportSummaryPivot `yaml:"report_summary_pivots"`
} `yaml:"settings"`
}

View File

@@ -676,7 +676,7 @@ func (c *CronTask) generateReport(ctx context.Context, tableName string) error {
dest := c.reportsDir()
start := time.Now()
c.Logger.Debug("Report generation start", "table", tableName, "dest", dest)
filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest)
filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest, c.Settings)
if err == nil {
c.Logger.Debug("Report generation complete", "table", tableName, "file", filename, "duration", time.Since(start))
} else {

View File

@@ -46,7 +46,7 @@ func (h *Handler) SnapshotRegenerateHourlyReports(w http.ResponseWriter, r *http
skipped++
continue
}
if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir); err != nil {
if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir, h.Settings); err != nil {
errors++
h.Logger.Warn("failed to regenerate hourly report", "table", rec.TableName, "error", err)
continue

View File

@@ -65,7 +65,7 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request)
return
}
reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName)
reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName, h.Settings)
if err != nil {
h.Logger.Error("Failed to create snapshot report", "error", err, "table", tableName)
writeJSONError(w, http.StatusInternalServerError, fmt.Sprintf("Unable to create snapshot report: '%s'", err))

View File

@@ -40,6 +40,25 @@ settings:
monthly_job_timeout_seconds: 1200
monthly_aggregation_granularity: "hourly"
monthly_aggregation_cron: "10 3 1 * *"
# Optional: override Summary worksheet pivot layout in daily/monthly XLSX reports.
# metric values: avg_vcpu, avg_ram, prorated_vm_count, vm_name_count
report_summary_pivots:
- metric: avg_vcpu
title: "Sum of Avg vCPUs"
pivot_name: "PivotAvgVcpu"
pivot_range: "Summary!A3:H40"
- metric: avg_ram
title: "Sum of Avg RAM"
pivot_name: "PivotAvgRam"
pivot_range: "Summary!J3:P40"
- metric: prorated_vm_count
title: "Sum of prorated VM count"
pivot_name: "PivotProratedVmCount"
pivot_range: "Summary!A63:H82"
- metric: vm_name_count
title: "Count of VM Name"
pivot_name: "PivotVmNameCount"
pivot_range: "Summary!J63:P82"
cleanup_job_timeout_seconds: 600
tenants_to_filter:
node_charge_clusters: