From 96567f6211ab1f1e283368fb5bc82023d203d1f7 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Thu, 15 Jan 2026 15:53:39 +1100 Subject: [PATCH] fix aggregation sql --- db/helpers.go | 86 ++++++++++++++++++++++++++++++++- internal/tasks/cronstatus.go | 9 ++-- scripts/drone.sh | 5 +- server/router/docs/docs.go | 77 ++++++++++++++++++++++++----- server/router/docs/swagger.json | 77 ++++++++++++++++++++++++----- server/router/docs/swagger.yaml | 69 +++++++++++++++++++++----- 6 files changed, 277 insertions(+), 46 deletions(-) diff --git a/db/helpers.go b/db/helpers.go index 6c7f894..ed98a72 100644 --- a/db/helpers.go +++ b/db/helpers.go @@ -513,7 +513,13 @@ func RefineCreationDeletionFromUnion(ctx context.Context, dbConn *sqlx.DB, summa if _, err := SafeTableName(summaryTable); err != nil { return err } - sql := fmt.Sprintf(` + + driver := strings.ToLower(dbConn.DriverName()) + var sql string + + switch driver { + case "pgx", "postgres": + sql = fmt.Sprintf(` WITH snapshots AS ( %s ), timeline AS ( @@ -560,6 +566,84 @@ WHERE dst."Vcenter" = t."Vcenter" OR (dst."Name" IS NOT DISTINCT FROM t."Name") ); `, unionQuery, summaryTable) + default: + // SQLite variant (no FROM in UPDATE, no IS NOT DISTINCT FROM). Uses positional args to avoid placeholder count issues. + sql = fmt.Sprintf(` +WITH snapshots AS ( +%[1]s +), timeline AS ( + SELECT + s."VmId", + s."VmUuid", + s."Name", + s."Vcenter", + MIN(NULLIF(s."CreationTime", 0)) AS any_creation, + MIN(s."SnapshotTime") AS first_seen, + MAX(s."SnapshotTime") AS last_seen + FROM snapshots s + GROUP BY s."VmId", s."VmUuid", s."Name", s."Vcenter" +), enriched AS ( + SELECT + tl.*, + ( + SELECT MIN(s2."SnapshotTime") + FROM snapshots s2 + WHERE s2."Vcenter" = tl."Vcenter" + AND COALESCE(s2."VmId", '') = COALESCE(tl."VmId", '') + AND s2."SnapshotTime" > tl.last_seen + ) AS first_after + FROM timeline tl +) +UPDATE %[2]s +SET + "CreationTime" = COALESCE( + ( + SELECT CASE + WHEN t.any_creation IS NOT NULL AND t.any_creation > 0 AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.any_creation) > t.any_creation THEN t.any_creation + WHEN t.any_creation IS NULL AND t.first_seen IS NOT NULL AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.first_seen) > t.first_seen THEN t.first_seen + ELSE NULL + END + FROM enriched t + WHERE %[2]s."Vcenter" = t."Vcenter" AND ( + (%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR + (%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR + (%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR + (%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR + (%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name") + ) + LIMIT 1 + ), + "CreationTime" + ), + "DeletionTime" = COALESCE( + ( + SELECT t.first_after + FROM enriched t + WHERE %[2]s."Vcenter" = t."Vcenter" AND ( + (%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR + (%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR + (%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR + (%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR + (%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name") + ) + AND t.first_after IS NOT NULL + AND ("DeletionTime" IS NULL OR "DeletionTime" = 0 OR t.first_after < "DeletionTime") + LIMIT 1 + ), + "DeletionTime" + ) +WHERE EXISTS ( + SELECT 1 FROM enriched t + WHERE %[2]s."Vcenter" = t."Vcenter" AND ( + (%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR + (%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR + (%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR + (%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR + (%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name") + ) +); +`, unionQuery, summaryTable) + } _, err := dbConn.ExecContext(ctx, sql) return err diff --git a/internal/tasks/cronstatus.go b/internal/tasks/cronstatus.go index d9aa1a3..23a0c8a 100644 --- a/internal/tasks/cronstatus.go +++ b/internal/tasks/cronstatus.go @@ -2,7 +2,6 @@ package tasks import ( "context" - "database/sql" "time" "vctp/db" @@ -102,11 +101,11 @@ func (c *CronTracker) finish(ctx context.Context, job string, startedAt int64, r if err != nil { return err } - var lastError sql.NullString + lastErr := "" if runErr != nil { - lastError = sql.NullString{String: runErr.Error(), Valid: true} + lastErr = runErr.Error() } - err = upsertCronFinish(tx, c.bindType, job, startedAt, duration, lastError.String) + err = upsertCronFinish(tx, c.bindType, job, duration, lastErr) if err != nil { tx.Rollback() return err @@ -134,7 +133,7 @@ WHERE job_name = ? return err } -func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, startedAt int64, durationMS int64, lastErr string) error { +func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, durationMS int64, lastErr string) error { query := ` UPDATE cron_status SET ended_at = ?, duration_ms = ?, last_error = ?, in_progress = FALSE diff --git a/scripts/drone.sh b/scripts/drone.sh index 51b9f83..36d40ba 100755 --- a/scripts/drone.sh +++ b/scripts/drone.sh @@ -14,11 +14,10 @@ host_os=$(uname -s | tr '[:upper:]' '[:lower:]') host_arch=$(uname -m) platforms=("linux/amd64") if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then - platforms+=("darwin/amd64") + platforms=("darwin/amd64") fi -#platforms=("linux/amd64") -echo Building:: +echo Building: $package_name echo - Version $package_version echo - Commit $commit echo - Build Time $buildtime diff --git a/server/router/docs/docs.go b/server/router/docs/docs.go index 6dcf8fd..a36075e 100644 --- a/server/router/docs/docs.go +++ b/server/router/docs/docs.go @@ -43,14 +43,15 @@ const docTemplate = `{ }, "/api/cleanup/updates": { "delete": { - "description": "Removes update records that are no longer associated with a VM.", + "description": "Deprecated: Removes update records that are no longer associated with a VM.", "produces": [ "text/plain" ], "tags": [ "maintenance" ], - "summary": "Cleanup updates", + "summary": "Cleanup updates (deprecated)", + "deprecated": true, "responses": { "200": { "description": "Cleanup completed", @@ -69,14 +70,15 @@ const docTemplate = `{ }, "/api/cleanup/vcenter": { "delete": { - "description": "Removes all inventory entries associated with a vCenter URL.", + "description": "Deprecated: Removes all inventory entries associated with a vCenter URL.", "produces": [ "application/json" ], "tags": [ "maintenance" ], - "summary": "Cleanup vCenter inventory", + "summary": "Cleanup vCenter inventory (deprecated)", + "deprecated": true, "parameters": [ { "type": "string", @@ -159,7 +161,7 @@ const docTemplate = `{ }, "/api/event/vm/create": { "post": { - "description": "Parses a VM create CloudEvent and stores the event data.", + "description": "Deprecated: Parses a VM create CloudEvent and stores the event data.", "consumes": [ "application/json" ], @@ -169,7 +171,8 @@ const docTemplate = `{ "tags": [ "events" ], - "summary": "Record VM create event", + "summary": "Record VM create event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -205,7 +208,7 @@ const docTemplate = `{ }, "/api/event/vm/delete": { "post": { - "description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", + "description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", "consumes": [ "application/json" ], @@ -215,7 +218,8 @@ const docTemplate = `{ "tags": [ "events" ], - "summary": "Record VM delete event", + "summary": "Record VM delete event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -251,7 +255,7 @@ const docTemplate = `{ }, "/api/event/vm/modify": { "post": { - "description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", + "description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", "consumes": [ "application/json" ], @@ -261,7 +265,8 @@ const docTemplate = `{ "tags": [ "events" ], - "summary": "Record VM modify event", + "summary": "Record VM modify event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -306,7 +311,7 @@ const docTemplate = `{ }, "/api/event/vm/move": { "post": { - "description": "Parses a VM move CloudEvent and creates an update record.", + "description": "Deprecated: Parses a VM move CloudEvent and creates an update record.", "consumes": [ "application/json" ], @@ -316,7 +321,8 @@ const docTemplate = `{ "tags": [ "events" ], - "summary": "Record VM move event", + "summary": "Record VM move event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -671,6 +677,53 @@ const docTemplate = `{ } } }, + "/api/snapshots/regenerate-hourly-reports": { + "post": { + "description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "Regenerate hourly snapshot reports", + "responses": { + "200": { + "description": "Regeneration summary", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/metrics": { + "get": { + "description": "Exposes Prometheus metrics for vctp.", + "produces": [ + "text/plain" + ], + "tags": [ + "metrics" + ], + "summary": "Prometheus metrics", + "responses": { + "200": { + "description": "Prometheus metrics" + } + } + } + }, "/snapshots/daily": { "get": { "description": "Lists daily summary snapshot tables.", diff --git a/server/router/docs/swagger.json b/server/router/docs/swagger.json index 4cd5b39..9b5c3b3 100644 --- a/server/router/docs/swagger.json +++ b/server/router/docs/swagger.json @@ -32,14 +32,15 @@ }, "/api/cleanup/updates": { "delete": { - "description": "Removes update records that are no longer associated with a VM.", + "description": "Deprecated: Removes update records that are no longer associated with a VM.", "produces": [ "text/plain" ], "tags": [ "maintenance" ], - "summary": "Cleanup updates", + "summary": "Cleanup updates (deprecated)", + "deprecated": true, "responses": { "200": { "description": "Cleanup completed", @@ -58,14 +59,15 @@ }, "/api/cleanup/vcenter": { "delete": { - "description": "Removes all inventory entries associated with a vCenter URL.", + "description": "Deprecated: Removes all inventory entries associated with a vCenter URL.", "produces": [ "application/json" ], "tags": [ "maintenance" ], - "summary": "Cleanup vCenter inventory", + "summary": "Cleanup vCenter inventory (deprecated)", + "deprecated": true, "parameters": [ { "type": "string", @@ -148,7 +150,7 @@ }, "/api/event/vm/create": { "post": { - "description": "Parses a VM create CloudEvent and stores the event data.", + "description": "Deprecated: Parses a VM create CloudEvent and stores the event data.", "consumes": [ "application/json" ], @@ -158,7 +160,8 @@ "tags": [ "events" ], - "summary": "Record VM create event", + "summary": "Record VM create event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -194,7 +197,7 @@ }, "/api/event/vm/delete": { "post": { - "description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", + "description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", "consumes": [ "application/json" ], @@ -204,7 +207,8 @@ "tags": [ "events" ], - "summary": "Record VM delete event", + "summary": "Record VM delete event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -240,7 +244,7 @@ }, "/api/event/vm/modify": { "post": { - "description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", + "description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", "consumes": [ "application/json" ], @@ -250,7 +254,8 @@ "tags": [ "events" ], - "summary": "Record VM modify event", + "summary": "Record VM modify event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -295,7 +300,7 @@ }, "/api/event/vm/move": { "post": { - "description": "Parses a VM move CloudEvent and creates an update record.", + "description": "Deprecated: Parses a VM move CloudEvent and creates an update record.", "consumes": [ "application/json" ], @@ -305,7 +310,8 @@ "tags": [ "events" ], - "summary": "Record VM move event", + "summary": "Record VM move event (deprecated)", + "deprecated": true, "parameters": [ { "description": "CloudEvent payload", @@ -660,6 +666,53 @@ } } }, + "/api/snapshots/regenerate-hourly-reports": { + "post": { + "description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.", + "produces": [ + "application/json" + ], + "tags": [ + "snapshots" + ], + "summary": "Regenerate hourly snapshot reports", + "responses": { + "200": { + "description": "Regeneration summary", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "500": { + "description": "Server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/metrics": { + "get": { + "description": "Exposes Prometheus metrics for vctp.", + "produces": [ + "text/plain" + ], + "tags": [ + "metrics" + ], + "summary": "Prometheus metrics", + "responses": { + "200": { + "description": "Prometheus metrics" + } + } + } + }, "/snapshots/daily": { "get": { "description": "Lists daily summary snapshot tables.", diff --git a/server/router/docs/swagger.yaml b/server/router/docs/swagger.yaml index 30845d2..33017ab 100644 --- a/server/router/docs/swagger.yaml +++ b/server/router/docs/swagger.yaml @@ -175,7 +175,9 @@ paths: - ui /api/cleanup/updates: delete: - description: Removes update records that are no longer associated with a VM. + deprecated: true + description: 'Deprecated: Removes update records that are no longer associated + with a VM.' produces: - text/plain responses: @@ -187,12 +189,14 @@ paths: description: Server error schema: type: string - summary: Cleanup updates + summary: Cleanup updates (deprecated) tags: - maintenance /api/cleanup/vcenter: delete: - description: Removes all inventory entries associated with a vCenter URL. + deprecated: true + description: 'Deprecated: Removes all inventory entries associated with a vCenter + URL.' parameters: - description: vCenter URL in: query @@ -214,7 +218,7 @@ paths: additionalProperties: type: string type: object - summary: Cleanup vCenter inventory + summary: Cleanup vCenter inventory (deprecated) tags: - maintenance /api/encrypt: @@ -253,7 +257,9 @@ paths: post: consumes: - application/json - description: Parses a VM create CloudEvent and stores the event data. + deprecated: true + description: 'Deprecated: Parses a VM create CloudEvent and stores the event + data.' parameters: - description: CloudEvent payload in: body @@ -276,14 +282,16 @@ paths: description: Server error schema: type: string - summary: Record VM create event + summary: Record VM create event (deprecated) tags: - events /api/event/vm/delete: post: consumes: - application/json - description: Parses a VM delete CloudEvent and marks the VM as deleted in inventory. + deprecated: true + description: 'Deprecated: Parses a VM delete CloudEvent and marks the VM as + deleted in inventory.' parameters: - description: CloudEvent payload in: body @@ -306,15 +314,16 @@ paths: description: Server error schema: type: string - summary: Record VM delete event + summary: Record VM delete event (deprecated) tags: - events /api/event/vm/modify: post: consumes: - application/json - description: Parses a VM modify CloudEvent and creates an update record when - relevant changes are detected. + deprecated: true + description: 'Deprecated: Parses a VM modify CloudEvent and creates an update + record when relevant changes are detected.' parameters: - description: CloudEvent payload in: body @@ -343,14 +352,16 @@ paths: additionalProperties: type: string type: object - summary: Record VM modify event + summary: Record VM modify event (deprecated) tags: - events /api/event/vm/move: post: consumes: - application/json - description: Parses a VM move CloudEvent and creates an update record. + deprecated: true + description: 'Deprecated: Parses a VM move CloudEvent and creates an update + record.' parameters: - description: CloudEvent payload in: body @@ -379,7 +390,7 @@ paths: additionalProperties: type: string type: object - summary: Record VM move event + summary: Record VM move event (deprecated) tags: - events /api/import/vm: @@ -590,6 +601,38 @@ paths: summary: Migrate snapshot registry tags: - snapshots + /api/snapshots/regenerate-hourly-reports: + post: + description: Regenerates XLSX reports for hourly snapshots when the report files + are missing or empty. + produces: + - application/json + responses: + "200": + description: Regeneration summary + schema: + additionalProperties: true + type: object + "500": + description: Server error + schema: + additionalProperties: + type: string + type: object + summary: Regenerate hourly snapshot reports + tags: + - snapshots + /metrics: + get: + description: Exposes Prometheus metrics for vctp. + produces: + - text/plain + responses: + "200": + description: Prometheus metrics + summary: Prometheus metrics + tags: + - metrics /snapshots/daily: get: description: Lists daily summary snapshot tables.