nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix aggregation logic
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-23 09:38:08 +11:00
parent 8a3481b966
commit 3e2d95d3b9
16 changed files with 384 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
internal/tasks/inventorySnapshots.go
View File

@@ -417,23 +417,60 @@ func buildUnionQuery(tables []string, columns []string, whereClause string) (str
return "", fmt.Errorf("no columns provided for union")
}
queries := make([]string, 0, len(tables))
columnList := strings.Join(columns, ", ")
for _, table := range tables {
safeName, err := db.SafeTableName(table)
if err != nil {
return "", err
const maxCompoundTerms = 450
if len(tables) <= maxCompoundTerms {
queries := make([]string, 0, len(tables))
for _, table := range tables {
safeName, err := db.SafeTableName(table)
if err != nil {
return "", err
}
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
}
queries = append(queries, query)
}
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
if len(queries) == 0 {
return "", fmt.Errorf("no valid tables provided for union")
}
queries = append(queries, query)
union := strings.Join(queries, "\nUNION ALL\n")
return fmt.Sprintf("SELECT * FROM (%s) AS union_all", union), nil
}
if len(queries) == 0 {
batches := make([]string, 0, (len(tables)/maxCompoundTerms)+1)
batchIndex := 0
for start := 0; start < len(tables); start += maxCompoundTerms {
end := start + maxCompoundTerms
if end > len(tables) {
end = len(tables)
}
queries := make([]string, 0, end-start)
for _, table := range tables[start:end] {
safeName, err := db.SafeTableName(table)
if err != nil {
return "", err
}
query := fmt.Sprintf("SELECT %s FROM %s", columnList, safeName)
if whereClause != "" {
query = fmt.Sprintf("%s WHERE %s", query, whereClause)
}
queries = append(queries, query)
}
if len(queries) == 0 {
continue
}
subUnion := strings.Join(queries, "\nUNION ALL\n")
batches = append(batches, fmt.Sprintf("SELECT * FROM (%s) AS batch_%d", subUnion, batchIndex))
batchIndex++
}
if len(batches) == 0 {
return "", fmt.Errorf("no valid tables provided for union")
}
return strings.Join(queries, "\nUNION ALL\n"), nil
outerUnion := strings.Join(batches, "\nUNION ALL\n")
return fmt.Sprintf("SELECT * FROM (%s) AS union_batches", outerUnion), nil
}
func templateExclusionFilter() string {
@@ -465,9 +502,21 @@ func truncateDate(t time.Time) time.Time {
func filterSnapshotsWithRows(ctx context.Context, dbConn *sqlx.DB, snapshots []report.SnapshotRecord) []report.SnapshotRecord {
filtered := snapshots[:0]
log := loggerFromCtx(ctx, nil)
for _, snapshot := range snapshots {
if rowsExist, err := db.TableHasRows(ctx, dbConn, snapshot.TableName); err == nil && rowsExist {
switch {
case snapshot.SnapshotCount > 0:
filtered = append(filtered, snapshot)
case snapshot.SnapshotCount == 0:
if log != nil {
log.Debug("skipping snapshot table with zero count", "table", snapshot.TableName, "snapshot_time", snapshot.SnapshotTime)
}
default:
if rowsExist, err := db.TableHasRows(ctx, dbConn, snapshot.TableName); err == nil && rowsExist {
filtered = append(filtered, snapshot)
} else if log != nil {
log.Debug("snapshot table probe empty or failed", "table", snapshot.TableName, "snapshot_time", snapshot.SnapshotTime, "error", err)
}
}
}
return filtered
@@ -1072,14 +1121,16 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
// If VM count dropped versus totals and we still haven't marked missing, try another comparison + wider event window.
if missingCount == 0 && prevVmCount.Valid && prevVmCount.Int64 > int64(totals.VmCount) {
// Fallback: compare against latest registered snapshot table.
if prevTable, err := latestHourlySnapshotBefore(ctx, dbConn, startTime, loggerFromCtx(ctx, c.Logger)); err == nil && prevTable != "" {
moreMissing := c.markMissingFromPrevious(ctx, dbConn, prevTable, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
if moreMissing > 0 {
missingCount += moreMissing
// Fallback: locate a previous table only if we didn't already find one.
if prevTableName == "" {
if prevTable, err := latestHourlySnapshotBefore(ctx, dbConn, startTime, loggerFromCtx(ctx, c.Logger)); err == nil && prevTable != "" {
moreMissing := c.markMissingFromPrevious(ctx, dbConn, prevTable, url, startTime, presentSnapshots, presentByUuid, presentByName, inventoryByVmID, inventoryByUuid, inventoryByName)
if moreMissing > 0 {
missingCount += moreMissing
}
// Reuse this table name for later snapshot lookups when correlating deletion events.
prevTableName = prevTable
}
// Reuse this table name for later snapshot lookups when correlating deletion events.
prevTableName = prevTable
}
freq := time.Duration(c.Settings.Values.Settings.VcenterInventorySnapshotSeconds) * time.Second
if freq <= 0 {