diff --git a/db/helpers.go b/db/helpers.go index a1f5295..e5d1f88 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -1546,8 +1546,12 @@ func UpdateSummaryPresenceByWindow(ctx context.Context, dbConn *sqlx.DB, summary query := fmt.Sprintf(` UPDATE %s SET "AvgIsPresent" = CASE - WHEN %s > %s THEN (CAST((%s - %s) AS REAL) / ?) - ELSE 0 + WHEN ("CreationTime" IS NOT NULL AND "CreationTime" > 0) OR ("DeletionTime" IS NOT NULL AND "DeletionTime" > 0) THEN + CASE + WHEN %s > %s THEN (CAST((%s - %s) AS REAL) / ?) + ELSE 0 + END + ELSE "AvgIsPresent" END `, summaryTable, endExpr, startExpr, endExpr, startExpr) query = dbConn.Rebind(query) @@ -1596,6 +1600,7 @@ UPDATE %s dst SET "CreationTime" = CASE WHEN t.any_creation IS NOT NULL AND t.any_creation > 0 THEN LEAST(COALESCE(NULLIF(dst."CreationTime", 0), t.any_creation), t.any_creation) + WHEN (dst."CreationTime" IS NULL OR dst."CreationTime" = 0) AND t.first_seen IS NOT NULL AND t.first_seen > 0 THEN t.first_seen ELSE dst."CreationTime" END, "DeletionTime" = CASE @@ -1657,6 +1662,7 @@ SET ( SELECT CASE WHEN t.any_creation IS NOT NULL AND t.any_creation > 0 AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.any_creation) > t.any_creation THEN t.any_creation + WHEN (%[2]s."CreationTime" IS NULL OR %[2]s."CreationTime" = 0) AND t.first_seen IS NOT NULL AND t.first_seen > 0 THEN t.first_seen ELSE NULL END FROM enriched t diff --git a/internal/tasks/dailyAggregate.go b/internal/tasks/dailyAggregate.go index 4711c08..738c705 100644 --- a/internal/tasks/dailyAggregate.go +++ b/internal/tasks/dailyAggregate.go @@ -173,6 +173,7 @@ func (c *CronTask) aggregateDailySummary(ctx context.Context, targetTime time.Ti c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable) } c.Logger.Debug("Counted daily summary rows", "table", summaryTable, "rows", rowCount, "duration", time.Since(rowCountStart)) + logMissingCreationSummary(ctx, c.Logger, c.Database, summaryTable, rowCount) registerStart := time.Now() c.Logger.Debug("Registering daily snapshot", "table", summaryTable, "date", dayStart.Format("2006-01-02"), "rows", rowCount) @@ -421,6 +422,7 @@ LIMIT 1 c.Logger.Warn("unable to count daily summary rows", "error", err, "table", summaryTable) } c.Logger.Debug("Counted daily summary rows", "table", summaryTable, "rows", rowCount, "duration", time.Since(rowCountStart)) + logMissingCreationSummary(ctx, c.Logger, c.Database, summaryTable, rowCount) registerStart := time.Now() c.Logger.Debug("Registering daily snapshot", "table", summaryTable, "date", dayStart.Format("2006-01-02"), "rows", rowCount) @@ -1010,6 +1012,106 @@ func btoi(b bool) int64 { return 0 } +func logMissingCreationSummary(ctx context.Context, logger *slog.Logger, database db.Database, summaryTable string, totalRows int64) { + if logger == nil { + logger = slog.Default() + } + if err := db.ValidateTableName(summaryTable); err != nil { + logger.Warn("daily summary creation diagnostics skipped (invalid table)", "table", summaryTable, "error", err) + return + } + if ctx == nil { + ctx = context.Background() + } + diagCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + dbConn := database.DB() + var missingTotal int64 + countQuery := fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE "CreationTime" IS NULL OR "CreationTime" = 0`, summaryTable) + if err := dbConn.GetContext(diagCtx, &missingTotal, countQuery); err != nil { + logger.Warn("daily summary creation diagnostics failed", "table", summaryTable, "error", err) + return + } + if missingTotal == 0 { + logger.Debug("daily summary creation diagnostics", "table", summaryTable, "missing_creation", 0) + return + } + missingPct := 0.0 + if totalRows > 0 { + missingPct = float64(missingTotal) * 100 / float64(totalRows) + } + logger.Warn("daily summary rows missing CreationTime", + "table", summaryTable, + "missing_count", missingTotal, + "total_rows", totalRows, + "missing_pct", missingPct, + ) + + byVcenterQuery := fmt.Sprintf(` +SELECT "Vcenter", COUNT(*) +FROM %s +WHERE "CreationTime" IS NULL OR "CreationTime" = 0 +GROUP BY "Vcenter" +ORDER BY COUNT(*) DESC +`, summaryTable) + if rows, err := dbConn.QueryxContext(diagCtx, byVcenterQuery); err != nil { + logger.Warn("daily summary creation diagnostics (by vcenter) failed", "table", summaryTable, "error", err) + } else { + for rows.Next() { + var vcenter string + var count int64 + if err := rows.Scan(&vcenter, &count); err != nil { + continue + } + logger.Warn("daily summary rows missing CreationTime by vcenter", "table", summaryTable, "vcenter", vcenter, "missing_count", count) + } + rows.Close() + if err := rows.Err(); err != nil { + logger.Warn("daily summary creation diagnostics (by vcenter) iteration failed", "table", summaryTable, "error", err) + } + } + + const sampleLimit = 10 + sampleQuery := fmt.Sprintf(` +SELECT "Vcenter","VmId","VmUuid","Name","SamplesPresent","AvgIsPresent","SnapshotTime" +FROM %s +WHERE "CreationTime" IS NULL OR "CreationTime" = 0 +ORDER BY "SamplesPresent" DESC +LIMIT %d +`, summaryTable, sampleLimit) + if rows, err := dbConn.QueryxContext(diagCtx, sampleQuery); err != nil { + logger.Warn("daily summary creation diagnostics (sample) failed", "table", summaryTable, "error", err) + } else { + for rows.Next() { + var ( + vcenter string + vmId, vmUuid sql.NullString + name sql.NullString + samplesPresent, snapshotTime sql.NullInt64 + avgIsPresent sql.NullFloat64 + ) + if err := rows.Scan(&vcenter, &vmId, &vmUuid, &name, &samplesPresent, &avgIsPresent, &snapshotTime); err != nil { + continue + } + logger.Debug("daily summary missing CreationTime sample", + "table", summaryTable, + "vcenter", vcenter, + "vm_id", vmId.String, + "vm_uuid", vmUuid.String, + "name", name.String, + "samples_present", samplesPresent.Int64, + "avg_is_present", avgIsPresent.Float64, + "snapshot_time", snapshotTime.Int64, + ) + } + rows.Close() + if err := rows.Err(); err != nil { + logger.Warn("daily summary creation diagnostics (sample) iteration failed", "table", summaryTable, "error", err) + } + } +} + // persistDailyRollup stores per-day aggregates into vm_daily_rollup to speed monthly aggregation. func (c *CronTask) persistDailyRollup(ctx context.Context, dayUnix int64, agg map[dailyAggKey]*dailyAggVal, totalSamples int, totalSamplesByVcenter map[string]int) error { dbConn := c.Database.DB() diff --git a/server/handler/dailyCreationDiagnostics.go b/server/handler/dailyCreationDiagnostics.go new file mode 100644 index 0000000..e3c15ba --- /dev/null +++ b/server/handler/dailyCreationDiagnostics.go @@ -0,0 +1,150 @@ +package handler + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + "vctp/db" + "vctp/server/models" +) + +// DailyCreationDiagnostics returns missing CreationTime diagnostics for a daily summary table. +// @Summary Daily summary CreationTime diagnostics +// @Description Returns counts of daily summary rows missing CreationTime and sample rows for the given date. +// @Tags diagnostics +// @Produce json +// @Param date query string true "Daily date (YYYY-MM-DD)" +// @Success 200 {object} models.DailyCreationDiagnosticsResponse "Diagnostics result" +// @Failure 400 {object} models.ErrorResponse "Invalid request" +// @Failure 404 {object} models.ErrorResponse "Summary not found" +// @Failure 500 {object} models.ErrorResponse "Server error" +// @Router /api/diagnostics/daily-creation [get] +func (h *Handler) DailyCreationDiagnostics(w http.ResponseWriter, r *http.Request) { + dateValue := strings.TrimSpace(r.URL.Query().Get("date")) + if dateValue == "" { + writeJSONError(w, http.StatusBadRequest, "date is required") + return + } + + loc := time.Now().Location() + parsed, err := time.ParseInLocation("2006-01-02", dateValue, loc) + if err != nil { + writeJSONError(w, http.StatusBadRequest, "date must be YYYY-MM-DD") + return + } + + tableName := fmt.Sprintf("inventory_daily_summary_%s", parsed.Format("20060102")) + if _, err := db.SafeTableName(tableName); err != nil { + writeJSONError(w, http.StatusBadRequest, "invalid summary table name") + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + dbConn := h.Database.DB() + if !db.TableExists(ctx, dbConn, tableName) { + writeJSONError(w, http.StatusNotFound, "daily summary table not found") + return + } + + var totalRows int64 + countQuery := fmt.Sprintf(`SELECT COUNT(*) FROM %s`, tableName) + if err := dbConn.GetContext(ctx, &totalRows, countQuery); err != nil { + h.Logger.Warn("daily creation diagnostics count failed", "table", tableName, "error", err) + writeJSONError(w, http.StatusInternalServerError, "failed to read summary rows") + return + } + + var missingTotal int64 + missingQuery := fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE "CreationTime" IS NULL OR "CreationTime" = 0`, tableName) + if err := dbConn.GetContext(ctx, &missingTotal, missingQuery); err != nil { + h.Logger.Warn("daily creation diagnostics missing count failed", "table", tableName, "error", err) + writeJSONError(w, http.StatusInternalServerError, "failed to read missing creation rows") + return + } + + missingPct := 0.0 + if totalRows > 0 { + missingPct = float64(missingTotal) * 100 / float64(totalRows) + } + + byVcenter := make([]models.DailyCreationMissingByVcenter, 0) + byVcenterQuery := fmt.Sprintf(` +SELECT "Vcenter", COUNT(*) AS missing_count +FROM %s +WHERE "CreationTime" IS NULL OR "CreationTime" = 0 +GROUP BY "Vcenter" +ORDER BY missing_count DESC +`, tableName) + if rows, err := dbConn.QueryxContext(ctx, byVcenterQuery); err != nil { + h.Logger.Warn("daily creation diagnostics by-vcenter failed", "table", tableName, "error", err) + } else { + for rows.Next() { + var vcenter string + var count int64 + if err := rows.Scan(&vcenter, &count); err != nil { + continue + } + byVcenter = append(byVcenter, models.DailyCreationMissingByVcenter{ + Vcenter: vcenter, + MissingCount: count, + }) + } + rows.Close() + } + + const sampleLimit = 10 + samples := make([]models.DailyCreationMissingSample, 0, sampleLimit) + sampleQuery := fmt.Sprintf(` +SELECT "Vcenter","VmId","VmUuid","Name","SamplesPresent","AvgIsPresent","SnapshotTime" +FROM %s +WHERE "CreationTime" IS NULL OR "CreationTime" = 0 +ORDER BY "SamplesPresent" DESC +LIMIT %d +`, tableName, sampleLimit) + if rows, err := dbConn.QueryxContext(ctx, sampleQuery); err != nil { + h.Logger.Warn("daily creation diagnostics sample failed", "table", tableName, "error", err) + } else { + for rows.Next() { + var ( + vcenter string + vmId, vmUuid, name sql.NullString + samplesPresent, snapshotTime sql.NullInt64 + avgIsPresent sql.NullFloat64 + ) + if err := rows.Scan(&vcenter, &vmId, &vmUuid, &name, &samplesPresent, &avgIsPresent, &snapshotTime); err != nil { + continue + } + samples = append(samples, models.DailyCreationMissingSample{ + Vcenter: vcenter, + VmId: vmId.String, + VmUuid: vmUuid.String, + Name: name.String, + SamplesPresent: samplesPresent.Int64, + AvgIsPresent: avgIsPresent.Float64, + SnapshotTime: snapshotTime.Int64, + }) + } + rows.Close() + } + + response := models.DailyCreationDiagnosticsResponse{ + Status: "OK", + Date: parsed.Format("2006-01-02"), + Table: tableName, + TotalRows: totalRows, + MissingCreationCount: missingTotal, + MissingCreationPct: missingPct, + MissingByVcenter: byVcenter, + Samples: samples, + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(response) +} diff --git a/server/models/diagnostics.go b/server/models/diagnostics.go new file mode 100644 index 0000000..6ffb655 --- /dev/null +++ b/server/models/diagnostics.go @@ -0,0 +1,30 @@ +package models + +// DailyCreationMissingByVcenter captures missing CreationTime counts per vCenter. +type DailyCreationMissingByVcenter struct { + Vcenter string `json:"vcenter"` + MissingCount int64 `json:"missing_count"` +} + +// DailyCreationMissingSample is a sample daily summary row missing CreationTime. +type DailyCreationMissingSample struct { + Vcenter string `json:"vcenter"` + VmId string `json:"vm_id,omitempty"` + VmUuid string `json:"vm_uuid,omitempty"` + Name string `json:"name,omitempty"` + SamplesPresent int64 `json:"samples_present"` + AvgIsPresent float64 `json:"avg_is_present"` + SnapshotTime int64 `json:"snapshot_time"` +} + +// DailyCreationDiagnosticsResponse describes missing CreationTime diagnostics for a daily summary table. +type DailyCreationDiagnosticsResponse struct { + Status string `json:"status"` + Date string `json:"date"` + Table string `json:"table"` + TotalRows int64 `json:"total_rows"` + MissingCreationCount int64 `json:"missing_creation_count"` + MissingCreationPct float64 `json:"missing_creation_pct"` + MissingByVcenter []DailyCreationMissingByVcenter `json:"missing_by_vcenter"` + Samples []DailyCreationMissingSample `json:"samples"` +} diff --git a/server/router/docs/swagger.json b/server/router/docs/swagger.json index 44ab259..a126b80 100644 --- a/server/router/docs/swagger.json +++ b/server/router/docs/swagger.json @@ -93,6 +93,53 @@ } } }, + "/api/diagnostics/daily-creation": { + "get": { + "description": "Returns counts of daily summary rows missing CreationTime and sample rows for the given date.", + "produces": [ + "application/json" + ], + "tags": [ + "diagnostics" + ], + "summary": "Daily summary CreationTime diagnostics", + "parameters": [ + { + "type": "string", + "description": "Daily date (YYYY-MM-DD)", + "name": "date", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "Diagnostics result", + "schema": { + "$ref": "#/definitions/models.DailyCreationDiagnosticsResponse" + } + }, + "400": { + "description": "Invalid request", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "404": { + "description": "Summary not found", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + }, + "500": { + "description": "Server error", + "schema": { + "$ref": "#/definitions/models.ErrorResponse" + } + } + } + } + }, "/api/encrypt": { "post": { "description": "Encrypts a plaintext value and returns the ciphertext.", @@ -1287,6 +1334,78 @@ "type": "string" } } + }, + "models.DailyCreationDiagnosticsResponse": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "date": { + "type": "string" + }, + "table": { + "type": "string" + }, + "total_rows": { + "type": "integer" + }, + "missing_creation_count": { + "type": "integer" + }, + "missing_creation_pct": { + "type": "number" + }, + "missing_by_vcenter": { + "type": "array", + "items": { + "$ref": "#/definitions/models.DailyCreationMissingByVcenter" + } + }, + "samples": { + "type": "array", + "items": { + "$ref": "#/definitions/models.DailyCreationMissingSample" + } + } + } + }, + "models.DailyCreationMissingByVcenter": { + "type": "object", + "properties": { + "vcenter": { + "type": "string" + }, + "missing_count": { + "type": "integer" + } + } + }, + "models.DailyCreationMissingSample": { + "type": "object", + "properties": { + "vcenter": { + "type": "string" + }, + "vm_id": { + "type": "string" + }, + "vm_uuid": { + "type": "string" + }, + "name": { + "type": "string" + }, + "samples_present": { + "type": "integer" + }, + "avg_is_present": { + "type": "number" + }, + "snapshot_time": { + "type": "integer" + } + } } } -} \ No newline at end of file +} diff --git a/server/router/docs/swagger.yaml b/server/router/docs/swagger.yaml index 948e5c4..84cd721 100644 --- a/server/router/docs/swagger.yaml +++ b/server/router/docs/swagger.yaml @@ -233,9 +233,87 @@ definitions: status: type: string type: object + models.DailyCreationDiagnosticsResponse: + properties: + status: + type: string + date: + type: string + table: + type: string + total_rows: + type: integer + missing_creation_count: + type: integer + missing_creation_pct: + type: number + missing_by_vcenter: + items: + $ref: '#/definitions/models.DailyCreationMissingByVcenter' + type: array + samples: + items: + $ref: '#/definitions/models.DailyCreationMissingSample' + type: array + type: object + models.DailyCreationMissingByVcenter: + properties: + vcenter: + type: string + missing_count: + type: integer + type: object + models.DailyCreationMissingSample: + properties: + vcenter: + type: string + vm_id: + type: string + vm_uuid: + type: string + name: + type: string + samples_present: + type: integer + avg_is_present: + type: number + snapshot_time: + type: integer + type: object info: contact: {} paths: + /api/diagnostics/daily-creation: + get: + description: Returns counts of daily summary rows missing CreationTime and sample rows for the given date. + parameters: + - description: Daily date (YYYY-MM-DD) + in: query + name: date + required: true + type: string + produces: + - application/json + responses: + "200": + description: Diagnostics result + schema: + $ref: '#/definitions/models.DailyCreationDiagnosticsResponse' + "400": + description: Invalid request + schema: + $ref: '#/definitions/models.ErrorResponse' + "404": + description: Summary not found + schema: + $ref: '#/definitions/models.ErrorResponse' + "500": + description: Server error + schema: + $ref: '#/definitions/models.ErrorResponse' + summary: Daily summary CreationTime diagnostics + tags: + - diagnostics /: get: description: Renders the main UI page. diff --git a/server/router/router.go b/server/router/router.go index 3d260b6..e379b55 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -68,6 +68,7 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st mux.HandleFunc("/api/snapshots/repair", h.SnapshotRepair) mux.HandleFunc("/api/snapshots/repair/all", h.SnapshotRepairSuite) mux.HandleFunc("/api/snapshots/regenerate-hourly-reports", h.SnapshotRegenerateHourlyReports) + mux.HandleFunc("/api/diagnostics/daily-creation", h.DailyCreationDiagnostics) mux.HandleFunc("/vm/trace", h.VmTrace) mux.HandleFunc("/vcenters", h.VcenterList) mux.HandleFunc("/vcenters/totals", h.VcenterTotals)