sql fix

internal/report/snapshots.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 	"vctp/db"
 
@@ -33,6 +34,8 @@ type SnapshotMigrationStats struct {
 	Errors            int
 }
 
+var hourlyTotalsQueryDumpOnce sync.Once
+
 func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) {
 	dbConn := database.DB()
 	driver := strings.ToLower(dbConn.DriverName())
@@ -1204,8 +1207,11 @@ func buildHourlyTotals(ctx context.Context, logger *slog.Logger, dbConn *sqlx.DB
 		startExpr := `CASE WHEN "CreationTime" IS NOT NULL AND "CreationTime" > 0 AND "CreationTime" > ? THEN "CreationTime" ELSE ? END`
 		endExpr := `CASE WHEN "DeletionTime" IS NOT NULL AND "DeletionTime" > 0 AND "DeletionTime" < ? THEN "DeletionTime" ELSE ? END`
 		overlapExpr := fmt.Sprintf(`CASE WHEN %s > %s THEN (CAST((%s - %s) AS REAL) / ?) ELSE 0 END`, endExpr, startExpr, endExpr, startExpr)
-		aggStartExpr := `CASE WHEN creation_time IS NOT NULL AND creation_time > 0 AND creation_time > ? THEN creation_time ELSE ? END`
+		missingStartExpr := `CASE WHEN COALESCE(prev_agg.creation_time, lifecycle.first_seen) IS NOT NULL AND COALESCE(prev_agg.creation_time, lifecycle.first_seen) > 0 AND COALESCE(prev_agg.creation_time, lifecycle.first_seen) > ? THEN COALESCE(prev_agg.creation_time, lifecycle.first_seen) ELSE ? END`
-		aggEndExpr := `CASE WHEN deletion_time IS NOT NULL AND deletion_time > 0 AND deletion_time < ? THEN deletion_time ELSE ? END`
+		missingEndExpr := `CASE WHEN COALESCE(lifecycle.deleted_at, prev_agg.deletion_time) IS NOT NULL AND COALESCE(lifecycle.deleted_at, prev_agg.deletion_time) > 0 AND COALESCE(lifecycle.deleted_at, prev_agg.deletion_time) < ? THEN COALESCE(lifecycle.deleted_at, prev_agg.deletion_time) ELSE ? END`
+		missingOverlapExpr := fmt.Sprintf(`CASE WHEN %s > %s THEN (CAST((%s - %s) AS REAL) / ?) ELSE 0 END`, missingEndExpr, missingStartExpr, missingEndExpr, missingStartExpr)
+		aggStartExpr := `CASE WHEN COALESCE(agg.creation_time, lifecycle.first_seen) IS NOT NULL AND COALESCE(agg.creation_time, lifecycle.first_seen) > 0 AND COALESCE(agg.creation_time, lifecycle.first_seen) > ? THEN COALESCE(agg.creation_time, lifecycle.first_seen) ELSE ? END`
+		aggEndExpr := `CASE WHEN COALESCE(lifecycle.deleted_at, agg.deletion_time) IS NOT NULL AND COALESCE(lifecycle.deleted_at, agg.deletion_time) > 0 AND COALESCE(lifecycle.deleted_at, agg.deletion_time) < ? THEN COALESCE(lifecycle.deleted_at, agg.deletion_time) ELSE ? END`
 		aggOverlapExpr := fmt.Sprintf(`CASE WHEN %s > %s THEN (CAST((%s - %s) AS REAL) / ?) ELSE 0 END`, aggEndExpr, aggStartExpr, aggEndExpr, aggStartExpr)
 		idExpr := `COALESCE(NULLIF("VmId", ''), NULLIF("VmUuid", ''), NULLIF("Name", ''), 'unknown')`
 		vmKeyExpr := fmt.Sprintf(`(%s || '|' || COALESCE("Vcenter", ''))`, idExpr)
@@ -1321,11 +1327,11 @@ diag AS (
 agg_diag AS (
   SELECT
     COUNT(*) AS agg_count,
-    COALESCE(SUM(CASE WHEN creation_time IS NULL OR creation_time = 0 THEN 1 ELSE 0 END), 0) AS missing_creation,
+    COALESCE(SUM(CASE WHEN agg_presence.creation_time IS NULL OR agg_presence.creation_time = 0 THEN 1 ELSE 0 END), 0) AS missing_creation,
-    COALESCE(SUM(CASE WHEN deletion_time IS NULL OR deletion_time = 0 THEN 1 ELSE 0 END), 0) AS missing_deletion,
+    COALESCE(SUM(CASE WHEN agg_presence.deletion_time IS NULL OR agg_presence.deletion_time = 0 THEN 1 ELSE 0 END), 0) AS missing_deletion,
-    COALESCE(SUM(CASE WHEN creation_time > ? AND creation_time < ? THEN 1 ELSE 0 END), 0) AS created_in_interval,
+    COALESCE(SUM(CASE WHEN agg_presence.creation_time > ? AND agg_presence.creation_time < ? THEN 1 ELSE 0 END), 0) AS created_in_interval,
-    COALESCE(SUM(CASE WHEN deletion_time > ? AND deletion_time < ? THEN 1 ELSE 0 END), 0) AS deleted_in_interval,
+    COALESCE(SUM(CASE WHEN agg_presence.deletion_time > ? AND agg_presence.deletion_time < ? THEN 1 ELSE 0 END), 0) AS deleted_in_interval,
-    COALESCE(SUM(CASE WHEN presence > 0 AND presence < 1 THEN 1 ELSE 0 END), 0) AS partial_presence
+    COALESCE(SUM(CASE WHEN agg_presence.presence > 0 AND agg_presence.presence < 1 THEN 1 ELSE 0 END), 0) AS partial_presence
   FROM agg_presence
 )
 SELECT
@@ -1353,7 +1359,7 @@ SELECT
   agg_diag.deleted_in_interval,
   agg_diag.partial_presence
 FROM diag, agg_diag
-`, vmKeyExpr, overlapExpr, selected.TableName, templateExclusionFilter(), vmKeyExpr, prevTableName, templateExclusionFilter(), aggOverlapExpr, aggOverlapExpr)
+`, vmKeyExpr, overlapExpr, selected.TableName, templateExclusionFilter(), vmKeyExpr, prevTableName, templateExclusionFilter(), missingOverlapExpr, aggOverlapExpr)
 		query = dbConn.Rebind(query)
 		var row struct {
 			VmCount          int64   `db:"vm_count"`
@@ -1395,6 +1401,19 @@ FROM diag, agg_diag
 		args = append(args, hourStartUnix, hourEndUnix)
 		args = append(args, hourStartUnix, hourEndUnix)
 		if err := dbConn.GetContext(ctx, &row, query, args...); err != nil {
+			hourlyTotalsQueryDumpOnce.Do(func() {
+				logger.Debug(
+					"hourly totals query debug",
+					"table", selected.TableName,
+					"prev_table", prevTableName,
+					"snapshot_time", selected.SnapshotTime.Format(time.RFC3339),
+					"interval_start", hourWindowStart.Format(time.RFC3339),
+					"interval_end", hourEnd.Format(time.RFC3339),
+					"query", query,
+					"args", args,
+					"error", err,
+				)
+			})
 			return nil, err
 		}
 		snapshotLag := selected.SnapshotTime.Sub(hourEnd)