Add support for customizable pivot titles and ranges in summary reports
Some checks failed
continuous-integration/drone/push Build was killed

This commit is contained in:
2026-02-18 11:59:22 +11:00
parent 92c6797f05
commit 29c277f863
8 changed files with 217 additions and 90 deletions

View File

@@ -279,6 +279,12 @@ Snapshots:
- `settings.hourly_index_max_age_days`: age gate for keeping per-hourly-table indexes (`-1` disables cleanup, `0` trims all) - `settings.hourly_index_max_age_days`: age gate for keeping per-hourly-table indexes (`-1` disables cleanup, `0` trims all)
- `settings.snapshot_cleanup_cron`: cron expression for cleanup job - `settings.snapshot_cleanup_cron`: cron expression for cleanup job
- `settings.reports_dir`: directory to store generated XLSX reports (default: `/var/lib/vctp/reports`) - `settings.reports_dir`: directory to store generated XLSX reports (default: `/var/lib/vctp/reports`)
- `settings.report_summary_pivots`: optional list to override Summary worksheet pivot titles/names/ranges in daily/monthly XLSX reports
- `metric`: one of `avg_vcpu`, `avg_ram`, `prorated_vm_count`, `vm_name_count`
- `title`: pivot title text shown on Summary sheet
- `pivot_name`: internal pivot table name in the XLSX workbook
- `pivot_range`: target range (for example `Summary!A3:H40` or `A3:H40`)
- `title_cell` (optional): explicit title cell; if omitted, derived from `pivot_range`
- `settings.hourly_snapshot_retry_seconds`: interval for retrying failed hourly snapshots (default: 300 seconds) - `settings.hourly_snapshot_retry_seconds`: interval for retrying failed hourly snapshots (default: 300 seconds)
- `settings.hourly_snapshot_max_retries`: maximum retry attempts per vCenter snapshot (default: 3) - `settings.hourly_snapshot_max_retries`: maximum retry attempts per vCenter snapshot (default: 3)

View File

@@ -15,6 +15,7 @@ import (
"sync" "sync"
"time" "time"
"vctp/db" "vctp/db"
"vctp/internal/settings"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
"github.com/xuri/excelize/v2" "github.com/xuri/excelize/v2"
@@ -545,7 +546,7 @@ func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName
} }
} }
func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string) ([]byte, error) { func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string, cfg *settings.Settings) ([]byte, error) {
if err := db.ValidateTableName(tableName); err != nil { if err := db.ValidateTableName(tableName); err != nil {
return nil, err return nil, err
} }
@@ -753,7 +754,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
for _, spec := range specs { for _, spec := range specs {
reportHeaders = append(reportHeaders, spec.Name) reportHeaders = append(reportHeaders, spec.Name)
} }
addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName) addSummaryPivotSheet(logger, xlsx, sheetName, reportHeaders, rowCount, tableName, cfg)
meta := reportMetadata{ meta := reportMetadata{
TableName: tableName, TableName: tableName,
@@ -787,7 +788,7 @@ func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Co
} }
// SaveTableReport renders a table report and writes it to the destination directory with a .xlsx extension. // SaveTableReport renders a table report and writes it to the destination directory with a .xlsx extension.
func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string) (string, error) { func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName, destDir string, cfg *settings.Settings) (string, error) {
if err := db.ValidateTableName(tableName); err != nil { if err := db.ValidateTableName(tableName); err != nil {
return "", err return "", err
} }
@@ -802,7 +803,7 @@ func SaveTableReport(logger *slog.Logger, Database db.Database, ctx context.Cont
} }
logger.Debug("Report directory ready", "dest", destDir) logger.Debug("Report directory ready", "dest", destDir)
data, err := CreateTableReport(logger, Database, ctx, tableName) data, err := CreateTableReport(logger, Database, ctx, tableName, cfg)
if err != nil { if err != nil {
logger.Warn("Report render failed", "table", tableName, "error", err) logger.Warn("Report render failed", "table", tableName, "error", err)
return "", err return "", err
@@ -890,6 +891,7 @@ func addTotalsChartSheet(logger *slog.Logger, database db.Database, ctx context.
} }
type summaryPivotSpec struct { type summaryPivotSpec struct {
Metric string
Title string Title string
TitleCell string TitleCell string
PivotName string PivotName string
@@ -900,7 +902,139 @@ type summaryPivotSpec struct {
DataSummary string DataSummary string
} }
func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string) { func defaultSummaryPivotSpecs(summarySheet string) []summaryPivotSpec {
return []summaryPivotSpec{
{
Metric: "avg_vcpu",
Title: "Sum of Avg vCPUs",
TitleCell: "A1",
PivotName: "PivotAvgVcpu",
PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet),
RowFields: []string{"Datacenter", "ResourcePool"},
DataField: "AvgVcpuCount",
DataName: "Sum of Avg vCPUs",
DataSummary: "Sum",
},
{
Metric: "avg_ram",
Title: "Sum of Avg RAM",
TitleCell: "J1",
PivotName: "PivotAvgRam",
PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgRamGB",
DataName: "Sum of Avg RAM",
DataSummary: "Sum",
},
{
Metric: "prorated_vm_count",
Title: "Sum of prorated VM count",
TitleCell: "A61",
PivotName: "PivotProratedVmCount",
PivotRange: fmt.Sprintf("%s!A63:H82", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgIsPresent",
DataName: "Sum of prorated VM count",
DataSummary: "Sum",
},
{
Metric: "vm_name_count",
Title: "Count of VM Name",
TitleCell: "J61",
PivotName: "PivotVmNameCount",
PivotRange: fmt.Sprintf("%s!J63:P82", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "Name",
DataName: "Count of VM Name",
DataSummary: "Count",
},
}
}
func normalizePivotRange(summarySheet, pivotRange string) string {
trimmed := strings.TrimSpace(pivotRange)
if trimmed == "" {
return ""
}
if strings.Contains(trimmed, "!") {
return trimmed
}
return fmt.Sprintf("%s!%s", summarySheet, trimmed)
}
func titleCellFromPivotRange(pivotRange, fallback string) string {
trimmed := strings.TrimSpace(pivotRange)
if trimmed == "" {
return fallback
}
if bang := strings.Index(trimmed, "!"); bang >= 0 {
trimmed = trimmed[bang+1:]
}
trimmed = strings.ReplaceAll(trimmed, "$", "")
startCell := trimmed
if parts := strings.Split(trimmed, ":"); len(parts) > 0 {
startCell = parts[0]
}
col, row, err := excelize.CellNameToCoordinates(startCell)
if err != nil {
return fallback
}
titleRow := row - 2
if titleRow < 1 {
titleRow = 1
}
cell, err := excelize.CoordinatesToCellName(col, titleRow)
if err != nil {
return fallback
}
return cell
}
func resolveSummaryPivotSpecs(cfg *settings.Settings, summarySheet string) []summaryPivotSpec {
specs := defaultSummaryPivotSpecs(summarySheet)
if cfg == nil || cfg.Values == nil || len(cfg.Values.Settings.ReportSummaryPivots) == 0 {
return specs
}
specByMetric := make(map[string]*summaryPivotSpec, len(specs))
metricOrder := make([]string, 0, len(specs))
for i := range specs {
metric := strings.TrimSpace(strings.ToLower(specs[i].Metric))
if metric == "" {
continue
}
specByMetric[metric] = &specs[i]
metricOrder = append(metricOrder, metric)
}
for i, custom := range cfg.Values.Settings.ReportSummaryPivots {
metric := strings.TrimSpace(strings.ToLower(custom.Metric))
if metric == "" && i < len(metricOrder) {
metric = metricOrder[i]
}
spec := specByMetric[metric]
if spec == nil {
continue
}
if title := strings.TrimSpace(custom.Title); title != "" {
spec.Title = title
}
if name := strings.TrimSpace(custom.PivotName); name != "" {
spec.PivotName = name
}
if rng := normalizePivotRange(summarySheet, custom.PivotRange); rng != "" {
spec.PivotRange = rng
}
if cell := strings.TrimSpace(custom.TitleCell); cell != "" {
spec.TitleCell = cell
} else {
spec.TitleCell = titleCellFromPivotRange(spec.PivotRange, spec.TitleCell)
}
}
return specs
}
func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet string, headers []string, rowCount int, tableName string, cfg *settings.Settings) {
if logger == nil { if logger == nil {
logger = slog.Default() logger = slog.Default()
} }
@@ -940,48 +1074,7 @@ func addSummaryPivotSheet(logger *slog.Logger, xlsx *excelize.File, dataSheet st
return header, ok return header, ok
} }
specs := []summaryPivotSpec{ specs := resolveSummaryPivotSpecs(cfg, summarySheet)
{
Title: "Sum of Avg vCPUs",
TitleCell: "A1",
PivotName: "PivotAvgVcpu",
PivotRange: fmt.Sprintf("%s!A3:H40", summarySheet),
RowFields: []string{"Datacenter", "ResourcePool"},
DataField: "AvgVcpuCount",
DataName: "Sum of Avg vCPUs",
DataSummary: "Sum",
},
{
Title: "Sum of Avg RAM",
TitleCell: "J1",
PivotName: "PivotAvgRam",
PivotRange: fmt.Sprintf("%s!J3:P40", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgRamGB",
DataName: "Sum of Avg RAM",
DataSummary: "Sum",
},
{
Title: "Sum of prorated VM count",
TitleCell: "A41",
PivotName: "PivotProratedVmCount",
PivotRange: fmt.Sprintf("%s!A43:H62", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "AvgIsPresent",
DataName: "Sum of prorated VM count",
DataSummary: "Sum",
},
{
Title: "Count of VM Name",
TitleCell: "J41",
PivotName: "PivotVmNameCount",
PivotRange: fmt.Sprintf("%s!J43:P62", summarySheet),
RowFields: []string{"Datacenter"},
DataField: "Name",
DataName: "Count of VM Name",
DataSummary: "Count",
},
}
for _, spec := range specs { for _, spec := range specs {
xlsx.SetCellValue(summarySheet, spec.TitleCell, spec.Title) xlsx.SetCellValue(summarySheet, spec.TitleCell, spec.Title)

View File

@@ -27,7 +27,7 @@ func TestAddSummaryPivotSheetCreatesPivotTables(t *testing.T) {
} }
logger := slog.New(slog.NewTextHandler(io.Discard, nil)) logger := slog.New(slog.NewTextHandler(io.Discard, nil))
addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215") addSummaryPivotSheet(logger, xlsx, dataSheet, headers, 1, "inventory_daily_summary_20260215", nil)
pivots, err := xlsx.GetPivotTables("Summary") pivots, err := xlsx.GetPivotTables("Summary")
if err != nil { if err != nil {

View File

@@ -24,48 +24,57 @@ type Settings struct {
Values *SettingsYML Values *SettingsYML
} }
type ReportSummaryPivot struct {
Metric string `yaml:"metric"`
Title string `yaml:"title"`
PivotName string `yaml:"pivot_name"`
PivotRange string `yaml:"pivot_range"`
TitleCell string `yaml:"title_cell"`
}
// SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties // SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties
type SettingsYML struct { type SettingsYML struct {
Settings struct { Settings struct {
LogLevel string `yaml:"log_level"` LogLevel string `yaml:"log_level"`
LogOutput string `yaml:"log_output"` LogOutput string `yaml:"log_output"`
DatabaseDriver string `yaml:"database_driver"` DatabaseDriver string `yaml:"database_driver"`
DatabaseURL string `yaml:"database_url"` DatabaseURL string `yaml:"database_url"`
EnableExperimentalPostgres bool `yaml:"enable_experimental_postgres"` EnableExperimentalPostgres bool `yaml:"enable_experimental_postgres"`
BindIP string `yaml:"bind_ip"` BindIP string `yaml:"bind_ip"`
BindPort int `yaml:"bind_port"` BindPort int `yaml:"bind_port"`
BindDisableTLS bool `yaml:"bind_disable_tls"` BindDisableTLS bool `yaml:"bind_disable_tls"`
TLSCertFilename string `yaml:"tls_cert_filename"` TLSCertFilename string `yaml:"tls_cert_filename"`
TLSKeyFilename string `yaml:"tls_key_filename"` TLSKeyFilename string `yaml:"tls_key_filename"`
EncryptionKey string `yaml:"encryption_key"` EncryptionKey string `yaml:"encryption_key"`
VcenterUsername string `yaml:"vcenter_username"` VcenterUsername string `yaml:"vcenter_username"`
VcenterPassword string `yaml:"vcenter_password"` VcenterPassword string `yaml:"vcenter_password"`
VcenterInsecure bool `yaml:"vcenter_insecure"` VcenterInsecure bool `yaml:"vcenter_insecure"`
EnableLegacyAPI bool `yaml:"enable_legacy_api"` EnableLegacyAPI bool `yaml:"enable_legacy_api"`
VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"` VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"`
VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"` VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"`
VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"` VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"`
VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"` VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"`
HourlySnapshotConcurrency int `yaml:"hourly_snapshot_concurrency"` HourlySnapshotConcurrency int `yaml:"hourly_snapshot_concurrency"`
HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"` HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"`
DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"` DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"`
HourlyIndexMaxAgeDays int `yaml:"hourly_index_max_age_days"` HourlyIndexMaxAgeDays int `yaml:"hourly_index_max_age_days"`
SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"` SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"`
ReportsDir string `yaml:"reports_dir"` ReportsDir string `yaml:"reports_dir"`
HourlyJobTimeoutSeconds int `yaml:"hourly_job_timeout_seconds"` HourlyJobTimeoutSeconds int `yaml:"hourly_job_timeout_seconds"`
HourlySnapshotTimeoutSeconds int `yaml:"hourly_snapshot_timeout_seconds"` HourlySnapshotTimeoutSeconds int `yaml:"hourly_snapshot_timeout_seconds"`
HourlySnapshotRetrySeconds int `yaml:"hourly_snapshot_retry_seconds"` HourlySnapshotRetrySeconds int `yaml:"hourly_snapshot_retry_seconds"`
HourlySnapshotMaxRetries int `yaml:"hourly_snapshot_max_retries"` HourlySnapshotMaxRetries int `yaml:"hourly_snapshot_max_retries"`
DailyJobTimeoutSeconds int `yaml:"daily_job_timeout_seconds"` DailyJobTimeoutSeconds int `yaml:"daily_job_timeout_seconds"`
MonthlyJobTimeoutSeconds int `yaml:"monthly_job_timeout_seconds"` MonthlyJobTimeoutSeconds int `yaml:"monthly_job_timeout_seconds"`
MonthlyAggregationGranularity string `yaml:"monthly_aggregation_granularity"` MonthlyAggregationGranularity string `yaml:"monthly_aggregation_granularity"`
MonthlyAggregationCron string `yaml:"monthly_aggregation_cron"` MonthlyAggregationCron string `yaml:"monthly_aggregation_cron"`
CleanupJobTimeoutSeconds int `yaml:"cleanup_job_timeout_seconds"` CleanupJobTimeoutSeconds int `yaml:"cleanup_job_timeout_seconds"`
TenantsToFilter []string `yaml:"tenants_to_filter"` TenantsToFilter []string `yaml:"tenants_to_filter"`
NodeChargeClusters []string `yaml:"node_charge_clusters"` NodeChargeClusters []string `yaml:"node_charge_clusters"`
SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"` SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"`
VcenterAddresses []string `yaml:"vcenter_addresses"` VcenterAddresses []string `yaml:"vcenter_addresses"`
PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"` PostgresWorkMemMB int `yaml:"postgres_work_mem_mb"`
ReportSummaryPivots []ReportSummaryPivot `yaml:"report_summary_pivots"`
} `yaml:"settings"` } `yaml:"settings"`
} }

View File

@@ -676,7 +676,7 @@ func (c *CronTask) generateReport(ctx context.Context, tableName string) error {
dest := c.reportsDir() dest := c.reportsDir()
start := time.Now() start := time.Now()
c.Logger.Debug("Report generation start", "table", tableName, "dest", dest) c.Logger.Debug("Report generation start", "table", tableName, "dest", dest)
filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest) filename, err := report.SaveTableReport(c.Logger, c.Database, ctx, tableName, dest, c.Settings)
if err == nil { if err == nil {
c.Logger.Debug("Report generation complete", "table", tableName, "file", filename, "duration", time.Since(start)) c.Logger.Debug("Report generation complete", "table", tableName, "file", filename, "duration", time.Since(start))
} else { } else {

View File

@@ -46,7 +46,7 @@ func (h *Handler) SnapshotRegenerateHourlyReports(w http.ResponseWriter, r *http
skipped++ skipped++
continue continue
} }
if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir); err != nil { if _, err := report.SaveTableReport(h.Logger, h.Database, ctx, rec.TableName, reportsDir, h.Settings); err != nil {
errors++ errors++
h.Logger.Warn("failed to regenerate hourly report", "table", rec.TableName, "error", err) h.Logger.Warn("failed to regenerate hourly report", "table", rec.TableName, "error", err)
continue continue

View File

@@ -65,7 +65,7 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request)
return return
} }
reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName) reportData, err := report.CreateTableReport(h.Logger, h.Database, ctx, tableName, h.Settings)
if err != nil { if err != nil {
h.Logger.Error("Failed to create snapshot report", "error", err, "table", tableName) h.Logger.Error("Failed to create snapshot report", "error", err, "table", tableName)
writeJSONError(w, http.StatusInternalServerError, fmt.Sprintf("Unable to create snapshot report: '%s'", err)) writeJSONError(w, http.StatusInternalServerError, fmt.Sprintf("Unable to create snapshot report: '%s'", err))

View File

@@ -40,6 +40,25 @@ settings:
monthly_job_timeout_seconds: 1200 monthly_job_timeout_seconds: 1200
monthly_aggregation_granularity: "hourly" monthly_aggregation_granularity: "hourly"
monthly_aggregation_cron: "10 3 1 * *" monthly_aggregation_cron: "10 3 1 * *"
# Optional: override Summary worksheet pivot layout in daily/monthly XLSX reports.
# metric values: avg_vcpu, avg_ram, prorated_vm_count, vm_name_count
report_summary_pivots:
- metric: avg_vcpu
title: "Sum of Avg vCPUs"
pivot_name: "PivotAvgVcpu"
pivot_range: "Summary!A3:H40"
- metric: avg_ram
title: "Sum of Avg RAM"
pivot_name: "PivotAvgRam"
pivot_range: "Summary!J3:P40"
- metric: prorated_vm_count
title: "Sum of prorated VM count"
pivot_name: "PivotProratedVmCount"
pivot_range: "Summary!A63:H82"
- metric: vm_name_count
title: "Count of VM Name"
pivot_name: "PivotVmNameCount"
pivot_range: "Summary!J63:P82"
cleanup_job_timeout_seconds: 600 cleanup_job_timeout_seconds: 600
tenants_to_filter: tenants_to_filter:
node_charge_clusters: node_charge_clusters: