enhance utilisation of postgres features

2026-04-20 10:19:27 +10:00
parent 98e92a8264
commit 8ccf5a7009
28 changed files with 2836 additions and 422 deletions
@@ -32,15 +32,15 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.
 		now := time.Now()
 		firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
 		targetMonth := firstOfThisMonth.AddDate(0, -1, 0)
-		return c.aggregateMonthlySummary(jobCtx, targetMonth, false)
+		return c.aggregateMonthlySummaryWithMode(jobCtx, targetMonth, false, true)
 	})
 }

 func (c *CronTask) AggregateMonthlySummary(ctx context.Context, month time.Time, force bool) error {
-	return c.aggregateMonthlySummary(ctx, month, force)
+	return c.aggregateMonthlySummaryWithMode(ctx, month, force, false)
 }

-func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time.Time, force bool) error {
+func (c *CronTask) aggregateMonthlySummaryWithMode(ctx context.Context, targetMonth time.Time, force bool, scheduled bool) error {
 	jobStart := time.Now()
 	if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
 		return err
@@ -48,11 +48,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time

 	granularity := strings.ToLower(strings.TrimSpace(c.Settings.Values.Settings.MonthlyAggregationGranularity))
 	if granularity == "" {
-		granularity = "hourly"
+		granularity = "daily"
+	}
+	if scheduled {
+		granularity = "daily"
 	}
 	if granularity != "hourly" && granularity != "daily" {
-		c.Logger.Warn("unknown monthly aggregation granularity; defaulting to hourly", "granularity", granularity)
-		granularity = "hourly"
+		c.Logger.Warn("unknown monthly aggregation granularity; defaulting to daily", "granularity", granularity)
+		granularity = "daily"
 	}

 	monthStart := time.Date(targetMonth.Year(), targetMonth.Month(), 1, 0, 0, 0, 0, targetMonth.Location())
@@ -60,7 +63,14 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
 	dbConn := c.Database.DB()
 	db.SetPostgresWorkMem(ctx, dbConn, c.Settings.Values.Settings.PostgresWorkMemMB)
 	driver := strings.ToLower(dbConn.DriverName())
-	useGoAgg := os.Getenv("MONTHLY_AGG_GO") == "1"
+	// Canonical Go aggregation is the default for both scheduled and manual runs.
+	// Legacy SQL/union aggregation stays available as a manual fallback/backfill path.
+	forceGoAgg := os.Getenv("MONTHLY_AGG_GO") == "1"
+	forceSQLAgg := !scheduled && os.Getenv("MONTHLY_AGG_SQL") == "1"
+	useGoAgg := scheduled || forceGoAgg || !forceSQLAgg
+	if forceSQLAgg && !forceGoAgg {
+		c.Logger.Info("MONTHLY_AGG_SQL=1 enabled; using SQL fallback path for manual monthly aggregation")
+	}
 	if !useGoAgg && granularity == "hourly" && driver == "sqlite" {
 		c.Logger.Warn("SQL monthly aggregation is slow on sqlite; overriding to Go path", "granularity", granularity)
 		useGoAgg = true
@@ -68,26 +78,28 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time

 	var snapshots []report.SnapshotRecord
 	var unionColumns []string
-	if granularity == "daily" {
-		dailySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd)
-		if err != nil {
-			return err
+	if !scheduled {
+		if granularity == "daily" {
+			dailySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "daily", "inventory_daily_summary_", "20060102", monthStart, monthEnd)
+			if err != nil {
+				return err
+			}
+			dailySnapshots = filterRecordsInRange(dailySnapshots, monthStart, monthEnd)
+			dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots)
+			snapshots = dailySnapshots
+			unionColumns = monthlyUnionColumns
+		} else {
+			hourlySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", monthStart, monthEnd)
+			if err != nil {
+				return err
+			}
+			hourlySnapshots = filterRecordsInRange(hourlySnapshots, monthStart, monthEnd)
+			hourlySnapshots = filterSnapshotsWithRows(ctx, dbConn, hourlySnapshots)
+			snapshots = hourlySnapshots
+			unionColumns = summaryUnionColumns
 		}
-		dailySnapshots = filterRecordsInRange(dailySnapshots, monthStart, monthEnd)
-		dailySnapshots = filterSnapshotsWithRows(ctx, dbConn, dailySnapshots)
-		snapshots = dailySnapshots
-		unionColumns = monthlyUnionColumns
-	} else {
-		hourlySnapshots, err := report.SnapshotRecordsWithFallback(ctx, c.Database, "hourly", "inventory_hourly_", "epoch", monthStart, monthEnd)
-		if err != nil {
-			return err
-		}
-		hourlySnapshots = filterRecordsInRange(hourlySnapshots, monthStart, monthEnd)
-		hourlySnapshots = filterSnapshotsWithRows(ctx, dbConn, hourlySnapshots)
-		snapshots = hourlySnapshots
-		unionColumns = summaryUnionColumns
 	}
-	if len(snapshots) == 0 {
+	if !scheduled && len(snapshots) == 0 {
 		return fmt.Errorf("no %s snapshot tables found for %s", granularity, targetMonth.Format("2006-01"))
 	}

@@ -110,12 +122,26 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
 		}
 	}

+	if scheduled && c.scheduledAggregationEngine() == "sql" {
+		c.Logger.Info("scheduled_aggregation_engine=sql enabled; using canonical SQL monthly aggregation path")
+		if err := c.aggregateMonthlySummarySQLCanonical(ctx, monthStart, monthEnd, monthlyTable); err != nil {
+			c.Logger.Warn("scheduled canonical SQL monthly aggregation failed; falling back to go path", "error", err)
+		} else {
+			metrics.RecordMonthlyAggregation(time.Since(jobStart), nil)
+			c.Logger.Debug("Finished monthly inventory aggregation (SQL canonical path)", "summary_table", monthlyTable)
+			return nil
+		}
+	}
+
 	// Optional Go-based aggregation path.
 	if useGoAgg {
 		switch granularity {
 		case "daily":
 			c.Logger.Debug("Using go implementation of monthly aggregation (daily)")
-			if err := c.aggregateMonthlySummaryGo(ctx, monthStart, monthEnd, monthlyTable, snapshots); err != nil {
+			if err := c.aggregateMonthlySummaryGo(ctx, monthStart, monthEnd, monthlyTable, snapshots, scheduled); err != nil {
+				if scheduled {
+					return err
+				}
 				c.Logger.Warn("go-based monthly aggregation failed, falling back to SQL path", "error", err)
 			} else {
 				metrics.RecordMonthlyAggregation(time.Since(jobStart), nil)
@@ -123,6 +149,9 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
 				return nil
 			}
 		case "hourly":
+			if scheduled {
+				return fmt.Errorf("scheduled monthly aggregation does not support hourly source mode")
+			}
 			c.Logger.Debug("Using go implementation of monthly aggregation (hourly)")
 			if err := c.aggregateMonthlySummaryGoHourly(ctx, monthStart, monthEnd, monthlyTable, snapshots); err != nil {
 				c.Logger.Warn("go-based monthly aggregation failed, falling back to SQL path", "error", err)
@@ -135,6 +164,9 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time
 			c.Logger.Warn("MONTHLY_AGG_GO is set but granularity is unsupported; using SQL path", "granularity", granularity)
 		}
 	}
+	if scheduled {
+		return fmt.Errorf("scheduled monthly aggregation requires go daily-rollup path")
+	}

 	tables := make([]string, 0, len(snapshots))
 	for _, snapshot := range snapshots {
@@ -190,7 +222,7 @@ func (c *CronTask) aggregateMonthlySummary(ctx context.Context, targetMonth time

 	db.AnalyzeTableIfPostgres(ctx, dbConn, monthlyTable)

-	if err := c.generateReport(ctx, monthlyTable); err != nil {
+	if err := c.generateReportWithPolicy(ctx, monthlyTable); err != nil {
 		c.Logger.Warn("failed to generate monthly report", "error", err, "table", monthlyTable)
 		metrics.RecordMonthlyAggregation(time.Since(jobStart), err)
 		return err
@@ -205,6 +237,52 @@ func monthlySummaryTableName(t time.Time) (string, error) {
 	return db.SafeTableName(fmt.Sprintf("inventory_monthly_summary_%s", t.Format("200601")))
 }

+func (c *CronTask) aggregateMonthlySummarySQLCanonical(ctx context.Context, monthStart, monthEnd time.Time, summaryTable string) error {
+	jobStart := time.Now()
+	dbConn := c.Database.DB()
+	if !db.TableExists(ctx, dbConn, "vm_daily_rollup") {
+		return fmt.Errorf("vm_daily_rollup table not found for canonical SQL monthly aggregation")
+	}
+
+	unionQuery := buildCanonicalDailyRollupSummaryUnion(monthStart, monthEnd)
+	insertQuery, err := db.BuildMonthlySummaryInsert(summaryTable, unionQuery)
+	if err != nil {
+		return err
+	}
+	if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil {
+		return err
+	}
+	if applied, err := db.ApplyLifecycleDeletionToSummary(ctx, dbConn, summaryTable, monthStart.Unix(), monthEnd.Unix()); err != nil {
+		c.Logger.Warn("failed to apply lifecycle deletions to monthly summary (SQL canonical)", "error", err, "table", summaryTable)
+	} else {
+		c.Logger.Info("Monthly aggregation deletion times", "source_lifecycle_cache", applied)
+	}
+	if err := db.RefineCreationDeletionFromUnion(ctx, dbConn, summaryTable, buildDailyRollupLifecycleUnion(monthStart, monthEnd)); err != nil {
+		c.Logger.Warn("failed to refine creation/deletion times (monthly SQL canonical)", "error", err, "table", summaryTable)
+	}
+	if err := db.UpdateSummaryPresenceByWindow(ctx, dbConn, summaryTable, monthStart.Unix(), monthEnd.Unix()); err != nil {
+		c.Logger.Warn("failed to update monthly AvgIsPresent from lifecycle window (SQL canonical)", "error", err, "table", summaryTable)
+	}
+
+	db.AnalyzeTableIfPostgres(ctx, dbConn, summaryTable)
+	rowCount, err := db.TableRowCount(ctx, dbConn, summaryTable)
+	if err != nil {
+		c.Logger.Warn("unable to count monthly summary rows (SQL canonical)", "error", err, "table", summaryTable)
+	}
+	if rowCount == 0 {
+		return fmt.Errorf("no VM records aggregated for %s", monthStart.Format("2006-01"))
+	}
+	if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil {
+		c.Logger.Warn("failed to register monthly snapshot (SQL canonical)", "error", err, "table", summaryTable)
+	}
+	if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil {
+		c.Logger.Warn("failed to generate monthly report (SQL canonical)", "error", err, "table", summaryTable)
+		return err
+	}
+	c.Logger.Debug("Finished monthly inventory aggregation (SQL canonical path)", "summary_table", summaryTable, "duration", time.Since(jobStart))
+	return nil
+}
+
 // aggregateMonthlySummaryGoHourly aggregates hourly snapshots directly into the monthly summary table.
 func (c *CronTask) aggregateMonthlySummaryGoHourly(ctx context.Context, monthStart, monthEnd time.Time, summaryTable string, hourlySnapshots []report.SnapshotRecord) error {
 	jobStart := time.Now()
@@ -311,7 +389,7 @@ func (c *CronTask) aggregateMonthlySummaryGoHourly(ctx context.Context, monthSta
 	if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil {
 		c.Logger.Warn("failed to register monthly snapshot (Go hourly)", "error", err, "table", summaryTable)
 	}
-	if err := c.generateReport(ctx, summaryTable); err != nil {
+	if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil {
 		c.Logger.Warn("failed to generate monthly report (Go hourly)", "error", err, "table", summaryTable)
 		return err
 	}
@@ -328,7 +406,7 @@ func (c *CronTask) aggregateMonthlySummaryGoHourly(ctx context.Context, monthSta

 // aggregateMonthlySummaryGo mirrors the SQL-based monthly aggregation but performs the work in Go,
 // reading daily summaries in parallel and reducing them to a single monthly summary table.
-func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, monthEnd time.Time, summaryTable string, dailySnapshots []report.SnapshotRecord) error {
+func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, monthEnd time.Time, summaryTable string, dailySnapshots []report.SnapshotRecord, canonicalOnly bool) error {
 	jobStart := time.Now()
 	dbConn := c.Database.DB()

@@ -336,26 +414,39 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo
 		return err
 	}

-	// Build union query for lifecycle refinement after inserts.
-	dailyTables := make([]string, 0, len(dailySnapshots))
-	for _, snapshot := range dailySnapshots {
-		dailyTables = append(dailyTables, snapshot.TableName)
-	}
-	unionQuery, err := buildUnionQuery(dailyTables, monthlyUnionColumns, templateExclusionFilter())
-	if err != nil {
-		return err
-	}
+	unionQuery := ""
+	var (
+		aggMap map[monthlyAggKey]*monthlyAggVal
+		err    error
+	)
+	if canonicalOnly {
+		aggMap, err = c.scanDailyRollup(ctx, monthStart, monthEnd)
+		if err != nil {
+			return err
+		}
+		unionQuery = buildDailyRollupLifecycleUnion(monthStart, monthEnd)
+	} else {
+		// Build union query for lifecycle refinement after inserts.
+		dailyTables := make([]string, 0, len(dailySnapshots))
+		for _, snapshot := range dailySnapshots {
+			dailyTables = append(dailyTables, snapshot.TableName)
+		}
+		unionQuery, err = buildUnionQuery(dailyTables, monthlyUnionColumns, templateExclusionFilter())
+		if err != nil {
+			return err
+		}

-	aggMap, err := c.scanDailyTablesParallel(ctx, dailySnapshots)
-	if err != nil {
-		return err
-	}
-	if len(aggMap) == 0 {
-		cacheAgg, cacheErr := c.scanDailyRollup(ctx, monthStart, monthEnd)
-		if cacheErr == nil && len(cacheAgg) > 0 {
-			aggMap = cacheAgg
-		} else if cacheErr != nil {
-			c.Logger.Warn("failed to read daily rollup cache; using table scan", "error", cacheErr)
+		aggMap, err = c.scanDailyTablesParallel(ctx, dailySnapshots)
+		if err != nil {
+			return err
+		}
+		if len(aggMap) == 0 {
+			cacheAgg, cacheErr := c.scanDailyRollup(ctx, monthStart, monthEnd)
+			if cacheErr == nil && len(cacheAgg) > 0 {
+				aggMap = cacheAgg
+			} else if cacheErr != nil {
+				c.Logger.Warn("failed to read daily rollup cache; using table scan", "error", cacheErr)
+			}
 		}
 	}
 	if len(aggMap) == 0 {
@@ -387,7 +478,7 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo
 	if err := report.RegisterSnapshot(ctx, c.Database, "monthly", summaryTable, monthStart, rowCount); err != nil {
 		c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", summaryTable)
 	}
-	if err := c.generateReport(ctx, summaryTable); err != nil {
+	if err := c.generateReportWithPolicy(ctx, summaryTable); err != nil {
 		c.Logger.Warn("failed to generate monthly report (Go)", "error", err, "table", summaryTable)
 		return err
 	}
@@ -666,6 +757,58 @@ WHERE "Date" >= ? AND "Date" < ?
 	return agg, rows.Err()
 }

+func buildDailyRollupLifecycleUnion(start, end time.Time) string {
+	return fmt.Sprintf(`
+SELECT
+  "VmId","VmUuid","Name","Vcenter","CreationTime","DeletionTime","Date" AS "SnapshotTime"
+FROM vm_daily_rollup
+WHERE "Date" >= %d AND "Date" < %d
+`, start.Unix(), end.Unix())
+}
+
+func buildCanonicalDailyRollupSummaryUnion(start, end time.Time) string {
+	return fmt.Sprintf(`
+SELECT
+  NULL AS "InventoryId",
+  COALESCE("Name",'') AS "Name",
+  COALESCE("Vcenter",'') AS "Vcenter",
+  COALESCE("VmId",'') AS "VmId",
+  NULL AS "EventKey",
+  NULL AS "CloudId",
+  COALESCE("CreationTime",0) AS "CreationTime",
+  COALESCE("DeletionTime",0) AS "DeletionTime",
+  COALESCE("LastResourcePool",'') AS "ResourcePool",
+  COALESCE("LastDatacenter",'') AS "Datacenter",
+  COALESCE("LastCluster",'') AS "Cluster",
+  COALESCE("LastFolder",'') AS "Folder",
+  COALESCE("LastProvisionedDisk",0) AS "ProvisionedDisk",
+  COALESCE("LastVcpuCount",0) AS "VcpuCount",
+  COALESCE("LastRamGB",0) AS "RamGB",
+  COALESCE("IsTemplate",'') AS "IsTemplate",
+  COALESCE("PoweredOn",'') AS "PoweredOn",
+  COALESCE("SrmPlaceholder",'') AS "SrmPlaceholder",
+  COALESCE("VmUuid",'') AS "VmUuid",
+  COALESCE("SamplesPresent",0) AS "SamplesPresent",
+  CASE WHEN COALESCE("TotalSamples",0) > 0 THEN 1.0 * COALESCE("SumVcpu",0) / "TotalSamples" ELSE NULL END AS "AvgVcpuCount",
+  CASE WHEN COALESCE("TotalSamples",0) > 0 THEN 1.0 * COALESCE("SumRam",0) / "TotalSamples" ELSE NULL END AS "AvgRamGB",
+  CASE WHEN COALESCE("TotalSamples",0) > 0 THEN 1.0 * COALESCE("SumDisk",0) / "TotalSamples" ELSE NULL END AS "AvgProvisionedDisk",
+  CASE WHEN COALESCE("TotalSamples",0) > 0 THEN 1.0 * COALESCE("SamplesPresent",0) / "TotalSamples" ELSE NULL END AS "AvgIsPresent",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("TinHits",0) / "SamplesPresent" ELSE NULL END AS "PoolTinPct",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("BronzeHits",0) / "SamplesPresent" ELSE NULL END AS "PoolBronzePct",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("SilverHits",0) / "SamplesPresent" ELSE NULL END AS "PoolSilverPct",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("GoldHits",0) / "SamplesPresent" ELSE NULL END AS "PoolGoldPct",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("TinHits",0) / "SamplesPresent" ELSE NULL END AS "Tin",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("BronzeHits",0) / "SamplesPresent" ELSE NULL END AS "Bronze",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("SilverHits",0) / "SamplesPresent" ELSE NULL END AS "Silver",
+  CASE WHEN COALESCE("SamplesPresent",0) > 0 THEN 100.0 * COALESCE("GoldHits",0) / "SamplesPresent" ELSE NULL END AS "Gold",
+  "Date" AS "SnapshotTime"
+FROM vm_daily_rollup
+WHERE "Date" >= %d
+  AND "Date" < %d
+  AND %s
+`, start.Unix(), end.Unix(), templateExclusionFilter())
+}
+
 func (c *CronTask) insertMonthlyAggregates(ctx context.Context, summaryTable string, aggMap map[monthlyAggKey]*monthlyAggVal) error {
 	dbConn := c.Database.DB()
 	columns := []string{