fix aggregation sql
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
2026-01-15 15:53:39 +11:00
parent 7971098caf
commit 96567f6211
6 changed files with 277 additions and 46 deletions

View File

@@ -513,7 +513,13 @@ func RefineCreationDeletionFromUnion(ctx context.Context, dbConn *sqlx.DB, summa
if _, err := SafeTableName(summaryTable); err != nil { if _, err := SafeTableName(summaryTable); err != nil {
return err return err
} }
sql := fmt.Sprintf(`
driver := strings.ToLower(dbConn.DriverName())
var sql string
switch driver {
case "pgx", "postgres":
sql = fmt.Sprintf(`
WITH snapshots AS ( WITH snapshots AS (
%s %s
), timeline AS ( ), timeline AS (
@@ -560,6 +566,84 @@ WHERE dst."Vcenter" = t."Vcenter"
OR (dst."Name" IS NOT DISTINCT FROM t."Name") OR (dst."Name" IS NOT DISTINCT FROM t."Name")
); );
`, unionQuery, summaryTable) `, unionQuery, summaryTable)
default:
// SQLite variant (no FROM in UPDATE, no IS NOT DISTINCT FROM). Uses positional args to avoid placeholder count issues.
sql = fmt.Sprintf(`
WITH snapshots AS (
%[1]s
), timeline AS (
SELECT
s."VmId",
s."VmUuid",
s."Name",
s."Vcenter",
MIN(NULLIF(s."CreationTime", 0)) AS any_creation,
MIN(s."SnapshotTime") AS first_seen,
MAX(s."SnapshotTime") AS last_seen
FROM snapshots s
GROUP BY s."VmId", s."VmUuid", s."Name", s."Vcenter"
), enriched AS (
SELECT
tl.*,
(
SELECT MIN(s2."SnapshotTime")
FROM snapshots s2
WHERE s2."Vcenter" = tl."Vcenter"
AND COALESCE(s2."VmId", '') = COALESCE(tl."VmId", '')
AND s2."SnapshotTime" > tl.last_seen
) AS first_after
FROM timeline tl
)
UPDATE %[2]s
SET
"CreationTime" = COALESCE(
(
SELECT CASE
WHEN t.any_creation IS NOT NULL AND t.any_creation > 0 AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.any_creation) > t.any_creation THEN t.any_creation
WHEN t.any_creation IS NULL AND t.first_seen IS NOT NULL AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.first_seen) > t.first_seen THEN t.first_seen
ELSE NULL
END
FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
LIMIT 1
),
"CreationTime"
),
"DeletionTime" = COALESCE(
(
SELECT t.first_after
FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
AND t.first_after IS NOT NULL
AND ("DeletionTime" IS NULL OR "DeletionTime" = 0 OR t.first_after < "DeletionTime")
LIMIT 1
),
"DeletionTime"
)
WHERE EXISTS (
SELECT 1 FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
);
`, unionQuery, summaryTable)
}
_, err := dbConn.ExecContext(ctx, sql) _, err := dbConn.ExecContext(ctx, sql)
return err return err

View File

@@ -2,7 +2,6 @@ package tasks
import ( import (
"context" "context"
"database/sql"
"time" "time"
"vctp/db" "vctp/db"
@@ -102,11 +101,11 @@ func (c *CronTracker) finish(ctx context.Context, job string, startedAt int64, r
if err != nil { if err != nil {
return err return err
} }
var lastError sql.NullString lastErr := ""
if runErr != nil { if runErr != nil {
lastError = sql.NullString{String: runErr.Error(), Valid: true} lastErr = runErr.Error()
} }
err = upsertCronFinish(tx, c.bindType, job, startedAt, duration, lastError.String) err = upsertCronFinish(tx, c.bindType, job, duration, lastErr)
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
return err return err
@@ -134,7 +133,7 @@ WHERE job_name = ?
return err return err
} }
func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, startedAt int64, durationMS int64, lastErr string) error { func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, durationMS int64, lastErr string) error {
query := ` query := `
UPDATE cron_status UPDATE cron_status
SET ended_at = ?, duration_ms = ?, last_error = ?, in_progress = FALSE SET ended_at = ?, duration_ms = ?, last_error = ?, in_progress = FALSE

View File

@@ -14,11 +14,10 @@ host_os=$(uname -s | tr '[:upper:]' '[:lower:]')
host_arch=$(uname -m) host_arch=$(uname -m)
platforms=("linux/amd64") platforms=("linux/amd64")
if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then
platforms+=("darwin/amd64") platforms=("darwin/amd64")
fi fi
#platforms=("linux/amd64")
echo Building:: echo Building: $package_name
echo - Version $package_version echo - Version $package_version
echo - Commit $commit echo - Commit $commit
echo - Build Time $buildtime echo - Build Time $buildtime

View File

@@ -43,14 +43,15 @@ const docTemplate = `{
}, },
"/api/cleanup/updates": { "/api/cleanup/updates": {
"delete": { "delete": {
"description": "Removes update records that are no longer associated with a VM.", "description": "Deprecated: Removes update records that are no longer associated with a VM.",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"maintenance" "maintenance"
], ],
"summary": "Cleanup updates", "summary": "Cleanup updates (deprecated)",
"deprecated": true,
"responses": { "responses": {
"200": { "200": {
"description": "Cleanup completed", "description": "Cleanup completed",
@@ -69,14 +70,15 @@ const docTemplate = `{
}, },
"/api/cleanup/vcenter": { "/api/cleanup/vcenter": {
"delete": { "delete": {
"description": "Removes all inventory entries associated with a vCenter URL.", "description": "Deprecated: Removes all inventory entries associated with a vCenter URL.",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"maintenance" "maintenance"
], ],
"summary": "Cleanup vCenter inventory", "summary": "Cleanup vCenter inventory (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"type": "string", "type": "string",
@@ -159,7 +161,7 @@ const docTemplate = `{
}, },
"/api/event/vm/create": { "/api/event/vm/create": {
"post": { "post": {
"description": "Parses a VM create CloudEvent and stores the event data.", "description": "Deprecated: Parses a VM create CloudEvent and stores the event data.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -169,7 +171,8 @@ const docTemplate = `{
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM create event", "summary": "Record VM create event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -205,7 +208,7 @@ const docTemplate = `{
}, },
"/api/event/vm/delete": { "/api/event/vm/delete": {
"post": { "post": {
"description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", "description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -215,7 +218,8 @@ const docTemplate = `{
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM delete event", "summary": "Record VM delete event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -251,7 +255,7 @@ const docTemplate = `{
}, },
"/api/event/vm/modify": { "/api/event/vm/modify": {
"post": { "post": {
"description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", "description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -261,7 +265,8 @@ const docTemplate = `{
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM modify event", "summary": "Record VM modify event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -306,7 +311,7 @@ const docTemplate = `{
}, },
"/api/event/vm/move": { "/api/event/vm/move": {
"post": { "post": {
"description": "Parses a VM move CloudEvent and creates an update record.", "description": "Deprecated: Parses a VM move CloudEvent and creates an update record.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -316,7 +321,8 @@ const docTemplate = `{
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM move event", "summary": "Record VM move event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -671,6 +677,53 @@ const docTemplate = `{
} }
} }
}, },
"/api/snapshots/regenerate-hourly-reports": {
"post": {
"description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Regenerate hourly snapshot reports",
"responses": {
"200": {
"description": "Regeneration summary",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/metrics": {
"get": {
"description": "Exposes Prometheus metrics for vctp.",
"produces": [
"text/plain"
],
"tags": [
"metrics"
],
"summary": "Prometheus metrics",
"responses": {
"200": {
"description": "Prometheus metrics"
}
}
}
},
"/snapshots/daily": { "/snapshots/daily": {
"get": { "get": {
"description": "Lists daily summary snapshot tables.", "description": "Lists daily summary snapshot tables.",

View File

@@ -32,14 +32,15 @@
}, },
"/api/cleanup/updates": { "/api/cleanup/updates": {
"delete": { "delete": {
"description": "Removes update records that are no longer associated with a VM.", "description": "Deprecated: Removes update records that are no longer associated with a VM.",
"produces": [ "produces": [
"text/plain" "text/plain"
], ],
"tags": [ "tags": [
"maintenance" "maintenance"
], ],
"summary": "Cleanup updates", "summary": "Cleanup updates (deprecated)",
"deprecated": true,
"responses": { "responses": {
"200": { "200": {
"description": "Cleanup completed", "description": "Cleanup completed",
@@ -58,14 +59,15 @@
}, },
"/api/cleanup/vcenter": { "/api/cleanup/vcenter": {
"delete": { "delete": {
"description": "Removes all inventory entries associated with a vCenter URL.", "description": "Deprecated: Removes all inventory entries associated with a vCenter URL.",
"produces": [ "produces": [
"application/json" "application/json"
], ],
"tags": [ "tags": [
"maintenance" "maintenance"
], ],
"summary": "Cleanup vCenter inventory", "summary": "Cleanup vCenter inventory (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"type": "string", "type": "string",
@@ -148,7 +150,7 @@
}, },
"/api/event/vm/create": { "/api/event/vm/create": {
"post": { "post": {
"description": "Parses a VM create CloudEvent and stores the event data.", "description": "Deprecated: Parses a VM create CloudEvent and stores the event data.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -158,7 +160,8 @@
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM create event", "summary": "Record VM create event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -194,7 +197,7 @@
}, },
"/api/event/vm/delete": { "/api/event/vm/delete": {
"post": { "post": {
"description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.", "description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -204,7 +207,8 @@
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM delete event", "summary": "Record VM delete event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -240,7 +244,7 @@
}, },
"/api/event/vm/modify": { "/api/event/vm/modify": {
"post": { "post": {
"description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.", "description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -250,7 +254,8 @@
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM modify event", "summary": "Record VM modify event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -295,7 +300,7 @@
}, },
"/api/event/vm/move": { "/api/event/vm/move": {
"post": { "post": {
"description": "Parses a VM move CloudEvent and creates an update record.", "description": "Deprecated: Parses a VM move CloudEvent and creates an update record.",
"consumes": [ "consumes": [
"application/json" "application/json"
], ],
@@ -305,7 +310,8 @@
"tags": [ "tags": [
"events" "events"
], ],
"summary": "Record VM move event", "summary": "Record VM move event (deprecated)",
"deprecated": true,
"parameters": [ "parameters": [
{ {
"description": "CloudEvent payload", "description": "CloudEvent payload",
@@ -660,6 +666,53 @@
} }
} }
}, },
"/api/snapshots/regenerate-hourly-reports": {
"post": {
"description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Regenerate hourly snapshot reports",
"responses": {
"200": {
"description": "Regeneration summary",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/metrics": {
"get": {
"description": "Exposes Prometheus metrics for vctp.",
"produces": [
"text/plain"
],
"tags": [
"metrics"
],
"summary": "Prometheus metrics",
"responses": {
"200": {
"description": "Prometheus metrics"
}
}
}
},
"/snapshots/daily": { "/snapshots/daily": {
"get": { "get": {
"description": "Lists daily summary snapshot tables.", "description": "Lists daily summary snapshot tables.",

View File

@@ -175,7 +175,9 @@ paths:
- ui - ui
/api/cleanup/updates: /api/cleanup/updates:
delete: delete:
description: Removes update records that are no longer associated with a VM. deprecated: true
description: 'Deprecated: Removes update records that are no longer associated
with a VM.'
produces: produces:
- text/plain - text/plain
responses: responses:
@@ -187,12 +189,14 @@ paths:
description: Server error description: Server error
schema: schema:
type: string type: string
summary: Cleanup updates summary: Cleanup updates (deprecated)
tags: tags:
- maintenance - maintenance
/api/cleanup/vcenter: /api/cleanup/vcenter:
delete: delete:
description: Removes all inventory entries associated with a vCenter URL. deprecated: true
description: 'Deprecated: Removes all inventory entries associated with a vCenter
URL.'
parameters: parameters:
- description: vCenter URL - description: vCenter URL
in: query in: query
@@ -214,7 +218,7 @@ paths:
additionalProperties: additionalProperties:
type: string type: string
type: object type: object
summary: Cleanup vCenter inventory summary: Cleanup vCenter inventory (deprecated)
tags: tags:
- maintenance - maintenance
/api/encrypt: /api/encrypt:
@@ -253,7 +257,9 @@ paths:
post: post:
consumes: consumes:
- application/json - application/json
description: Parses a VM create CloudEvent and stores the event data. deprecated: true
description: 'Deprecated: Parses a VM create CloudEvent and stores the event
data.'
parameters: parameters:
- description: CloudEvent payload - description: CloudEvent payload
in: body in: body
@@ -276,14 +282,16 @@ paths:
description: Server error description: Server error
schema: schema:
type: string type: string
summary: Record VM create event summary: Record VM create event (deprecated)
tags: tags:
- events - events
/api/event/vm/delete: /api/event/vm/delete:
post: post:
consumes: consumes:
- application/json - application/json
description: Parses a VM delete CloudEvent and marks the VM as deleted in inventory. deprecated: true
description: 'Deprecated: Parses a VM delete CloudEvent and marks the VM as
deleted in inventory.'
parameters: parameters:
- description: CloudEvent payload - description: CloudEvent payload
in: body in: body
@@ -306,15 +314,16 @@ paths:
description: Server error description: Server error
schema: schema:
type: string type: string
summary: Record VM delete event summary: Record VM delete event (deprecated)
tags: tags:
- events - events
/api/event/vm/modify: /api/event/vm/modify:
post: post:
consumes: consumes:
- application/json - application/json
description: Parses a VM modify CloudEvent and creates an update record when deprecated: true
relevant changes are detected. description: 'Deprecated: Parses a VM modify CloudEvent and creates an update
record when relevant changes are detected.'
parameters: parameters:
- description: CloudEvent payload - description: CloudEvent payload
in: body in: body
@@ -343,14 +352,16 @@ paths:
additionalProperties: additionalProperties:
type: string type: string
type: object type: object
summary: Record VM modify event summary: Record VM modify event (deprecated)
tags: tags:
- events - events
/api/event/vm/move: /api/event/vm/move:
post: post:
consumes: consumes:
- application/json - application/json
description: Parses a VM move CloudEvent and creates an update record. deprecated: true
description: 'Deprecated: Parses a VM move CloudEvent and creates an update
record.'
parameters: parameters:
- description: CloudEvent payload - description: CloudEvent payload
in: body in: body
@@ -379,7 +390,7 @@ paths:
additionalProperties: additionalProperties:
type: string type: string
type: object type: object
summary: Record VM move event summary: Record VM move event (deprecated)
tags: tags:
- events - events
/api/import/vm: /api/import/vm:
@@ -590,6 +601,38 @@ paths:
summary: Migrate snapshot registry summary: Migrate snapshot registry
tags: tags:
- snapshots - snapshots
/api/snapshots/regenerate-hourly-reports:
post:
description: Regenerates XLSX reports for hourly snapshots when the report files
are missing or empty.
produces:
- application/json
responses:
"200":
description: Regeneration summary
schema:
additionalProperties: true
type: object
"500":
description: Server error
schema:
additionalProperties:
type: string
type: object
summary: Regenerate hourly snapshot reports
tags:
- snapshots
/metrics:
get:
description: Exposes Prometheus metrics for vctp.
produces:
- text/plain
responses:
"200":
description: Prometheus metrics
summary: Prometheus metrics
tags:
- metrics
/snapshots/daily: /snapshots/daily:
get: get:
description: Lists daily summary snapshot tables. description: Lists daily summary snapshot tables.