nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix aggregation sql
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-15 15:53:39 +11:00
parent 7971098caf
commit 96567f6211
6 changed files with 277 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
db/helpers.go
View File

@@ -513,7 +513,13 @@ func RefineCreationDeletionFromUnion(ctx context.Context, dbConn *sqlx.DB, summa
if _, err := SafeTableName(summaryTable); err != nil {
return err
}
sql := fmt.Sprintf(`
driver := strings.ToLower(dbConn.DriverName())
var sql string
switch driver {
case "pgx", "postgres":
sql = fmt.Sprintf(`
WITH snapshots AS (
%s
), timeline AS (
@@ -560,6 +566,84 @@ WHERE dst."Vcenter" = t."Vcenter"
OR (dst."Name" IS NOT DISTINCT FROM t."Name")
);
`, unionQuery, summaryTable)
default:
// SQLite variant (no FROM in UPDATE, no IS NOT DISTINCT FROM). Uses positional args to avoid placeholder count issues.
sql = fmt.Sprintf(`
WITH snapshots AS (
%[1]s
), timeline AS (
SELECT
s."VmId",
s."VmUuid",
s."Name",
s."Vcenter",
MIN(NULLIF(s."CreationTime", 0)) AS any_creation,
MIN(s."SnapshotTime") AS first_seen,
MAX(s."SnapshotTime") AS last_seen
FROM snapshots s
GROUP BY s."VmId", s."VmUuid", s."Name", s."Vcenter"
), enriched AS (
SELECT
tl.*,
(
SELECT MIN(s2."SnapshotTime")
FROM snapshots s2
WHERE s2."Vcenter" = tl."Vcenter"
AND COALESCE(s2."VmId", '') = COALESCE(tl."VmId", '')
AND s2."SnapshotTime" > tl.last_seen
) AS first_after
FROM timeline tl
)
UPDATE %[2]s
SET
"CreationTime" = COALESCE(
(
SELECT CASE
WHEN t.any_creation IS NOT NULL AND t.any_creation > 0 AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.any_creation) > t.any_creation THEN t.any_creation
WHEN t.any_creation IS NULL AND t.first_seen IS NOT NULL AND COALESCE(NULLIF(%[2]s."CreationTime", 0), t.first_seen) > t.first_seen THEN t.first_seen
ELSE NULL
END
FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
LIMIT 1
),
"CreationTime"
),
"DeletionTime" = COALESCE(
(
SELECT t.first_after
FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
AND t.first_after IS NOT NULL
AND ("DeletionTime" IS NULL OR "DeletionTime" = 0 OR t.first_after < "DeletionTime")
LIMIT 1
),
"DeletionTime"
)
WHERE EXISTS (
SELECT 1 FROM enriched t
WHERE %[2]s."Vcenter" = t."Vcenter" AND (
(%[2]s."VmId" IS NOT NULL AND t."VmId" IS NOT NULL AND %[2]s."VmId" = t."VmId") OR
(%[2]s."VmId" IS NULL AND t."VmId" IS NULL) OR
(%[2]s."VmUuid" IS NOT NULL AND t."VmUuid" IS NOT NULL AND %[2]s."VmUuid" = t."VmUuid") OR
(%[2]s."VmUuid" IS NULL AND t."VmUuid" IS NULL) OR
(%[2]s."Name" IS NOT NULL AND t."Name" IS NOT NULL AND %[2]s."Name" = t."Name")
)
);
`, unionQuery, summaryTable)
}
_, err := dbConn.ExecContext(ctx, sql)
return err

9
internal/tasks/cronstatus.go
View File

@@ -2,7 +2,6 @@ package tasks
import (
"context"
"database/sql"
"time"
"vctp/db"
@@ -102,11 +101,11 @@ func (c *CronTracker) finish(ctx context.Context, job string, startedAt int64, r
if err != nil {
return err
}
var lastError sql.NullString
lastErr := ""
if runErr != nil {
lastError = sql.NullString{String: runErr.Error(), Valid: true}
lastErr = runErr.Error()
}
err = upsertCronFinish(tx, c.bindType, job, startedAt, duration, lastError.String)
err = upsertCronFinish(tx, c.bindType, job, duration, lastErr)
if err != nil {
tx.Rollback()
return err
@@ -134,7 +133,7 @@ WHERE job_name = ?
return err
}
func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, startedAt int64, durationMS int64, lastErr string) error {
func upsertCronFinish(tx *sqlx.Tx, bindType int, job string, durationMS int64, lastErr string) error {
query := `
UPDATE cron_status
SET ended_at = ?, duration_ms = ?, last_error = ?, in_progress = FALSE

5
scripts/drone.sh
View File

@@ -14,11 +14,10 @@ host_os=$(uname -s | tr '[:upper:]' '[:lower:]')
host_arch=$(uname -m)
platforms=("linux/amd64")
if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then
platforms+=("darwin/amd64")
platforms=("darwin/amd64")
fi
#platforms=("linux/amd64")
echo Building::
echo Building: $package_name
echo - Version $package_version
echo - Commit $commit
echo - Build Time $buildtime

77
server/router/docs/docs.go
View File

@@ -43,14 +43,15 @@ const docTemplate = `{
},
"/api/cleanup/updates": {
"delete": {
"description": "Removes update records that are no longer associated with a VM.",
"description": "Deprecated: Removes update records that are no longer associated with a VM.",
"produces": [
"text/plain"
],
"tags": [
"maintenance"
],
"summary": "Cleanup updates",
"summary": "Cleanup updates (deprecated)",
"deprecated": true,
"responses": {
"200": {
"description": "Cleanup completed",
@@ -69,14 +70,15 @@ const docTemplate = `{
},
"/api/cleanup/vcenter": {
"delete": {
"description": "Removes all inventory entries associated with a vCenter URL.",
"description": "Deprecated: Removes all inventory entries associated with a vCenter URL.",
"produces": [
"application/json"
],
"tags": [
"maintenance"
],
"summary": "Cleanup vCenter inventory",
"summary": "Cleanup vCenter inventory (deprecated)",
"deprecated": true,
"parameters": [
{
"type": "string",
@@ -159,7 +161,7 @@ const docTemplate = `{
},
"/api/event/vm/create": {
"post": {
"description": "Parses a VM create CloudEvent and stores the event data.",
"description": "Deprecated: Parses a VM create CloudEvent and stores the event data.",
"consumes": [
"application/json"
],
@@ -169,7 +171,8 @@ const docTemplate = `{
"tags": [
"events"
],
"summary": "Record VM create event",
"summary": "Record VM create event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -205,7 +208,7 @@ const docTemplate = `{
},
"/api/event/vm/delete": {
"post": {
"description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"consumes": [
"application/json"
],
@@ -215,7 +218,8 @@ const docTemplate = `{
"tags": [
"events"
],
"summary": "Record VM delete event",
"summary": "Record VM delete event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -251,7 +255,7 @@ const docTemplate = `{
},
"/api/event/vm/modify": {
"post": {
"description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"consumes": [
"application/json"
],
@@ -261,7 +265,8 @@ const docTemplate = `{
"tags": [
"events"
],
"summary": "Record VM modify event",
"summary": "Record VM modify event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -306,7 +311,7 @@ const docTemplate = `{
},
"/api/event/vm/move": {
"post": {
"description": "Parses a VM move CloudEvent and creates an update record.",
"description": "Deprecated: Parses a VM move CloudEvent and creates an update record.",
"consumes": [
"application/json"
],
@@ -316,7 +321,8 @@ const docTemplate = `{
"tags": [
"events"
],
"summary": "Record VM move event",
"summary": "Record VM move event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -671,6 +677,53 @@ const docTemplate = `{
}
}
},
"/api/snapshots/regenerate-hourly-reports": {
"post": {
"description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Regenerate hourly snapshot reports",
"responses": {
"200": {
"description": "Regeneration summary",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/metrics": {
"get": {
"description": "Exposes Prometheus metrics for vctp.",
"produces": [
"text/plain"
],
"tags": [
"metrics"
],
"summary": "Prometheus metrics",
"responses": {
"200": {
"description": "Prometheus metrics"
}
}
}
},
"/snapshots/daily": {
"get": {
"description": "Lists daily summary snapshot tables.",

77
server/router/docs/swagger.json
View File

@@ -32,14 +32,15 @@
},
"/api/cleanup/updates": {
"delete": {
"description": "Removes update records that are no longer associated with a VM.",
"description": "Deprecated: Removes update records that are no longer associated with a VM.",
"produces": [
"text/plain"
],
"tags": [
"maintenance"
],
"summary": "Cleanup updates",
"summary": "Cleanup updates (deprecated)",
"deprecated": true,
"responses": {
"200": {
"description": "Cleanup completed",
@@ -58,14 +59,15 @@
},
"/api/cleanup/vcenter": {
"delete": {
"description": "Removes all inventory entries associated with a vCenter URL.",
"description": "Deprecated: Removes all inventory entries associated with a vCenter URL.",
"produces": [
"application/json"
],
"tags": [
"maintenance"
],
"summary": "Cleanup vCenter inventory",
"summary": "Cleanup vCenter inventory (deprecated)",
"deprecated": true,
"parameters": [
{
"type": "string",
@@ -148,7 +150,7 @@
},
"/api/event/vm/create": {
"post": {
"description": "Parses a VM create CloudEvent and stores the event data.",
"description": "Deprecated: Parses a VM create CloudEvent and stores the event data.",
"consumes": [
"application/json"
],
@@ -158,7 +160,8 @@
"tags": [
"events"
],
"summary": "Record VM create event",
"summary": "Record VM create event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -194,7 +197,7 @@
},
"/api/event/vm/delete": {
"post": {
"description": "Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"description": "Deprecated: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.",
"consumes": [
"application/json"
],
@@ -204,7 +207,8 @@
"tags": [
"events"
],
"summary": "Record VM delete event",
"summary": "Record VM delete event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -240,7 +244,7 @@
},
"/api/event/vm/modify": {
"post": {
"description": "Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"description": "Deprecated: Parses a VM modify CloudEvent and creates an update record when relevant changes are detected.",
"consumes": [
"application/json"
],
@@ -250,7 +254,8 @@
"tags": [
"events"
],
"summary": "Record VM modify event",
"summary": "Record VM modify event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -295,7 +300,7 @@
},
"/api/event/vm/move": {
"post": {
"description": "Parses a VM move CloudEvent and creates an update record.",
"description": "Deprecated: Parses a VM move CloudEvent and creates an update record.",
"consumes": [
"application/json"
],
@@ -305,7 +310,8 @@
"tags": [
"events"
],
"summary": "Record VM move event",
"summary": "Record VM move event (deprecated)",
"deprecated": true,
"parameters": [
{
"description": "CloudEvent payload",
@@ -660,6 +666,53 @@
}
}
},
"/api/snapshots/regenerate-hourly-reports": {
"post": {
"description": "Regenerates XLSX reports for hourly snapshots when the report files are missing or empty.",
"produces": [
"application/json"
],
"tags": [
"snapshots"
],
"summary": "Regenerate hourly snapshot reports",
"responses": {
"200": {
"description": "Regeneration summary",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/metrics": {
"get": {
"description": "Exposes Prometheus metrics for vctp.",
"produces": [
"text/plain"
],
"tags": [
"metrics"
],
"summary": "Prometheus metrics",
"responses": {
"200": {
"description": "Prometheus metrics"
}
}
}
},
"/snapshots/daily": {
"get": {
"description": "Lists daily summary snapshot tables.",

69
server/router/docs/swagger.yaml
View File

@@ -175,7 +175,9 @@ paths:
- ui
/api/cleanup/updates:
delete:
description: Removes update records that are no longer associated with a VM.
deprecated: true
description: 'Deprecated: Removes update records that are no longer associated
with a VM.'
produces:
- text/plain
responses:
@@ -187,12 +189,14 @@ paths:
description: Server error
schema:
type: string
summary: Cleanup updates
summary: Cleanup updates (deprecated)
tags:
- maintenance
/api/cleanup/vcenter:
delete:
description: Removes all inventory entries associated with a vCenter URL.
deprecated: true
description: 'Deprecated: Removes all inventory entries associated with a vCenter
URL.'
parameters:
- description: vCenter URL
in: query
@@ -214,7 +218,7 @@ paths:
additionalProperties:
type: string
type: object
summary: Cleanup vCenter inventory
summary: Cleanup vCenter inventory (deprecated)
tags:
- maintenance
/api/encrypt:
@@ -253,7 +257,9 @@ paths:
post:
consumes:
- application/json
description: Parses a VM create CloudEvent and stores the event data.
deprecated: true
description: 'Deprecated: Parses a VM create CloudEvent and stores the event
data.'
parameters:
- description: CloudEvent payload
in: body
@@ -276,14 +282,16 @@ paths:
description: Server error
schema:
type: string
summary: Record VM create event
summary: Record VM create event (deprecated)
tags:
- events
/api/event/vm/delete:
post:
consumes:
- application/json
description: Parses a VM delete CloudEvent and marks the VM as deleted in inventory.
deprecated: true
description: 'Deprecated: Parses a VM delete CloudEvent and marks the VM as
deleted in inventory.'
parameters:
- description: CloudEvent payload
in: body
@@ -306,15 +314,16 @@ paths:
description: Server error
schema:
type: string
summary: Record VM delete event
summary: Record VM delete event (deprecated)
tags:
- events
/api/event/vm/modify:
post:
consumes:
- application/json
description: Parses a VM modify CloudEvent and creates an update record when
relevant changes are detected.
deprecated: true
description: 'Deprecated: Parses a VM modify CloudEvent and creates an update
record when relevant changes are detected.'
parameters:
- description: CloudEvent payload
in: body
@@ -343,14 +352,16 @@ paths:
additionalProperties:
type: string
type: object
summary: Record VM modify event
summary: Record VM modify event (deprecated)
tags:
- events
/api/event/vm/move:
post:
consumes:
- application/json
description: Parses a VM move CloudEvent and creates an update record.
deprecated: true
description: 'Deprecated: Parses a VM move CloudEvent and creates an update
record.'
parameters:
- description: CloudEvent payload
in: body
@@ -379,7 +390,7 @@ paths:
additionalProperties:
type: string
type: object
summary: Record VM move event
summary: Record VM move event (deprecated)
tags:
- events
/api/import/vm:
@@ -590,6 +601,38 @@ paths:
summary: Migrate snapshot registry
tags:
- snapshots
/api/snapshots/regenerate-hourly-reports:
post:
description: Regenerates XLSX reports for hourly snapshots when the report files
are missing or empty.
produces:
- application/json
responses:
"200":
description: Regeneration summary
schema:
additionalProperties: true
type: object
"500":
description: Server error
schema:
additionalProperties:
type: string
type: object
summary: Regenerate hourly snapshot reports
tags:
- snapshots
/metrics:
get:
description: Exposes Prometheus metrics for vctp.
produces:
- text/plain
responses:
"200":
description: Prometheus metrics
summary: Prometheus metrics
tags:
- metrics
/snapshots/daily:
get:
description: Lists daily summary snapshot tables.