Compare commits

...

2 Commits

Author SHA1 Message Date
7400e08c54 updates
All checks were successful
continuous-integration/drone/push Build is passing
2026-01-14 09:28:30 +11:00
ffe0c01fd7 improvements to inventory processing 2026-01-14 09:28:25 +11:00
36 changed files with 731 additions and 365 deletions

1
.gitignore vendored
View File

@@ -10,6 +10,7 @@
*.dylib
vctp
build/
settings.yaml
# Certificates
*.pem

View File

@@ -28,27 +28,37 @@ Run `templ generate -path ./components` to generate code based on template files
## Documentation
Run `swag init --exclude "pkg.mod,pkg.build,pkg.tools" -o server/router/docs`
#### Settings File
Configuration now lives in the YAML settings file. By default the service reads
`/etc/dtms/vctp.yml`, or you can override it with the `-settings` flag.
```shell
vctp -settings /path/to/vctp.yml
```
#### Database Configuration
By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL
by setting environment variables:
by updating the settings file:
- `DB_DRIVER`: `sqlite` (default) or `postgres`
- `DB_URL`: SQLite file path/DSN or PostgreSQL DSN
- `settings.database_driver`: `sqlite` (default) or `postgres`
- `settings.database_url`: SQLite file path/DSN or PostgreSQL DSN
Examples:
```shell
# SQLite (default)
DB_DRIVER=sqlite DB_URL=./db.sqlite3
```yaml
settings:
database_driver: sqlite
database_url: ./db.sqlite3
# PostgreSQL
DB_DRIVER=postgres DB_URL=postgres://user:pass@localhost:5432/vctp?sslmode=disable
settings:
database_driver: postgres
database_url: postgres://user:pass@localhost:5432/vctp?sslmode=disable
```
PostgreSQL migrations live in `db/migrations_postgres`, while SQLite migrations remain in
`db/migrations`.
#### Snapshot Retention
Hourly and daily snapshot table retention can be configured with environment variables:
Hourly and daily snapshot table retention can be configured in the settings file:
- `HOURLY_SNAPSHOT_MAX_AGE_DAYS` (default: 60)
- `DAILY_SNAPSHOT_MAX_AGE_MONTHS` (default: 12)
- `settings.hourly_snapshot_max_age_days` (default: 60)
- `settings.daily_snapshot_max_age_months` (default: 12)

View File

@@ -8,23 +8,25 @@ templ Header() {
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
<meta name="description" content="vCTP API endpoint"/>
<title>vCTP API</title>
<link rel="icon" href="/favicon.ico"/>
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"/>
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"/>
<script src="/assets/js/htmx@v2.0.2.min.js"></script>
<link href={ "/assets/css/output@" + version.Value + ".css" } rel="stylesheet"/>
<style>
:root {
--web2-blue: #3b82f6;
--web2-cyan: #22d3ee;
--web2-blue: #1d9bf0;
--web2-slate: #0f172a;
--web2-muted: #64748b;
--web2-card: #ffffff;
--web2-shadow: 0 20px 40px rgba(15, 23, 42, 0.15);
--web2-soft-shadow: 0 8px 20px rgba(15, 23, 42, 0.12);
--web2-border: #e5e7eb;
}
body {
font-family: "Trebuchet MS", "Lucida Grande", "Verdana", sans-serif;
font-family: "Segoe UI", "Helvetica Neue", Arial, sans-serif;
color: var(--web2-slate);
}
.web2-bg {
background: radial-gradient(circle at top left, #e0f2fe 0%, #f8fafc 45%, #e2e8f0 100%);
background: #ffffff;
}
.web2-shell {
max-width: 1100px;
@@ -32,29 +34,28 @@ templ Header() {
padding: 2rem 1.5rem 4rem;
}
.web2-header {
background: linear-gradient(135deg, #0ea5e9 0%, #6366f1 100%);
color: #fff;
border-radius: 22px;
box-shadow: var(--web2-shadow);
background: var(--web2-card);
border: 1px solid var(--web2-border);
border-radius: 4px;
padding: 1.5rem 2rem;
}
.web2-card {
background: var(--web2-card);
border-radius: 18px;
box-shadow: var(--web2-soft-shadow);
border: 1px solid var(--web2-border);
border-radius: 4px;
padding: 1.5rem 1.75rem;
}
.web2-pill {
display: inline-flex;
align-items: center;
gap: 0.4rem;
background: rgba(255, 255, 255, 0.2);
border: 1px solid rgba(255, 255, 255, 0.35);
padding: 0.35rem 0.8rem;
border-radius: 999px;
background: #f8fafc;
border: 1px solid var(--web2-border);
color: var(--web2-muted);
padding: 0.2rem 0.6rem;
border-radius: 3px;
font-size: 0.85rem;
letter-spacing: 0.02em;
backdrop-filter: blur(6px);
}
.web2-link {
color: var(--web2-blue);
@@ -65,22 +66,24 @@ templ Header() {
text-decoration: underline;
}
.web2-button {
background: linear-gradient(180deg, #60a5fa 0%, #2563eb 100%);
background: var(--web2-blue);
color: #fff;
padding: 0.5rem 1rem;
border-radius: 999px;
box-shadow: 0 6px 16px rgba(37, 99, 235, 0.35);
padding: 0.45rem 0.9rem;
border-radius: 3px;
border: 1px solid #1482d0;
box-shadow: none;
font-weight: 600;
text-decoration: none;
}
.web2-button:hover {
filter: brightness(1.05);
background: #1787d4;
}
.web2-list li {
background: rgba(255, 255, 255, 0.9);
border-radius: 14px;
padding: 0.85rem 1.1rem;
box-shadow: 0 6px 16px rgba(15, 23, 42, 0.08);
background: #ffffff;
border: 1px solid var(--web2-border);
border-radius: 3px;
padding: 0.75rem 1rem;
box-shadow: none;
}
</style>
</head>

View File

@@ -31,20 +31,20 @@ func Header() templ.Component {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<head><meta charset=\"UTF-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"><meta name=\"description\" content=\"vCTP API endpoint\"><title>vCTP API</title><script src=\"/assets/js/htmx@v2.0.2.min.js\"></script><link href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<head><meta charset=\"UTF-8\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\"><meta name=\"description\" content=\"vCTP API endpoint\"><title>vCTP API</title><link rel=\"icon\" href=\"/favicon.ico\"><link rel=\"icon\" type=\"image/png\" sizes=\"16x16\" href=\"/favicon-16x16.png\"><link rel=\"icon\" type=\"image/png\" sizes=\"32x32\" href=\"/favicon-32x32.png\"><script src=\"/assets/js/htmx@v2.0.2.min.js\"></script><link href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 templ.SafeURL
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinURLErrs("/assets/css/output@" + version.Value + ".css")
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/core/header.templ`, Line: 12, Col: 61}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `core/header.templ`, Line: 15, Col: 61}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\" rel=\"stylesheet\"><style>\n\t\t\t:root {\n\t\t\t\t--web2-blue: #3b82f6;\n\t\t\t\t--web2-cyan: #22d3ee;\n\t\t\t\t--web2-slate: #0f172a;\n\t\t\t\t--web2-card: #ffffff;\n\t\t\t\t--web2-shadow: 0 20px 40px rgba(15, 23, 42, 0.15);\n\t\t\t\t--web2-soft-shadow: 0 8px 20px rgba(15, 23, 42, 0.12);\n\t\t\t}\n\t\t\tbody {\n\t\t\t\tfont-family: \"Trebuchet MS\", \"Lucida Grande\", \"Verdana\", sans-serif;\n\t\t\t\tcolor: var(--web2-slate);\n\t\t\t}\n\t\t\t.web2-bg {\n\t\t\t\tbackground: radial-gradient(circle at top left, #e0f2fe 0%, #f8fafc 45%, #e2e8f0 100%);\n\t\t\t}\n\t\t\t.web2-shell {\n\t\t\t\tmax-width: 1100px;\n\t\t\t\tmargin: 0 auto;\n\t\t\t\tpadding: 2rem 1.5rem 4rem;\n\t\t\t}\n\t\t\t.web2-header {\n\t\t\t\tbackground: linear-gradient(135deg, #0ea5e9 0%, #6366f1 100%);\n\t\t\t\tcolor: #fff;\n\t\t\t\tborder-radius: 22px;\n\t\t\t\tbox-shadow: var(--web2-shadow);\n\t\t\t\tpadding: 1.5rem 2rem;\n\t\t\t}\n\t\t\t.web2-card {\n\t\t\t\tbackground: var(--web2-card);\n\t\t\t\tborder-radius: 18px;\n\t\t\t\tbox-shadow: var(--web2-soft-shadow);\n\t\t\t\tpadding: 1.5rem 1.75rem;\n\t\t\t}\n\t\t\t.web2-pill {\n\t\t\t\tdisplay: inline-flex;\n\t\t\t\talign-items: center;\n\t\t\t\tgap: 0.4rem;\n\t\t\t\tbackground: rgba(255, 255, 255, 0.2);\n\t\t\t\tborder: 1px solid rgba(255, 255, 255, 0.35);\n\t\t\t\tpadding: 0.35rem 0.8rem;\n\t\t\t\tborder-radius: 999px;\n\t\t\t\tfont-size: 0.85rem;\n\t\t\t\tletter-spacing: 0.02em;\n\t\t\t\tbackdrop-filter: blur(6px);\n\t\t\t}\n\t\t\t.web2-link {\n\t\t\t\tcolor: var(--web2-blue);\n\t\t\t\ttext-decoration: none;\n\t\t\t\tfont-weight: 600;\n\t\t\t}\n\t\t\t.web2-link:hover {\n\t\t\t\ttext-decoration: underline;\n\t\t\t}\n\t\t\t.web2-button {\n\t\t\t\tbackground: linear-gradient(180deg, #60a5fa 0%, #2563eb 100%);\n\t\t\t\tcolor: #fff;\n\t\t\t\tpadding: 0.5rem 1rem;\n\t\t\t\tborder-radius: 999px;\n\t\t\t\tbox-shadow: 0 6px 16px rgba(37, 99, 235, 0.35);\n\t\t\t\tfont-weight: 600;\n\t\t\t\ttext-decoration: none;\n\t\t\t}\n\t\t\t.web2-button:hover {\n\t\t\t\tfilter: brightness(1.05);\n\t\t\t}\n\t\t\t.web2-list li {\n\t\t\t\tbackground: rgba(255, 255, 255, 0.9);\n\t\t\t\tborder-radius: 14px;\n\t\t\t\tpadding: 0.85rem 1.1rem;\n\t\t\t\tbox-shadow: 0 6px 16px rgba(15, 23, 42, 0.08);\n\t\t\t}\n\t\t</style></head>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\" rel=\"stylesheet\"><style>\n\t\t\t:root {\n\t\t\t\t--web2-blue: #1d9bf0;\n\t\t\t\t--web2-slate: #0f172a;\n\t\t\t\t--web2-muted: #64748b;\n\t\t\t\t--web2-card: #ffffff;\n\t\t\t\t--web2-border: #e5e7eb;\n\t\t\t}\n\t\t\tbody {\n\t\t\t\tfont-family: \"Segoe UI\", \"Helvetica Neue\", Arial, sans-serif;\n\t\t\t\tcolor: var(--web2-slate);\n\t\t\t}\n\t\t\t.web2-bg {\n\t\t\t\tbackground: #ffffff;\n\t\t\t}\n\t\t\t.web2-shell {\n\t\t\t\tmax-width: 1100px;\n\t\t\t\tmargin: 0 auto;\n\t\t\t\tpadding: 2rem 1.5rem 4rem;\n\t\t\t}\n\t\t\t.web2-header {\n\t\t\t\tbackground: var(--web2-card);\n\t\t\t\tborder: 1px solid var(--web2-border);\n\t\t\t\tborder-radius: 4px;\n\t\t\t\tpadding: 1.5rem 2rem;\n\t\t\t}\n\t\t\t.web2-card {\n\t\t\t\tbackground: var(--web2-card);\n\t\t\t\tborder: 1px solid var(--web2-border);\n\t\t\t\tborder-radius: 4px;\n\t\t\t\tpadding: 1.5rem 1.75rem;\n\t\t\t}\n\t\t\t.web2-pill {\n\t\t\t\tdisplay: inline-flex;\n\t\t\t\talign-items: center;\n\t\t\t\tgap: 0.4rem;\n\t\t\t\tbackground: #f8fafc;\n\t\t\t\tborder: 1px solid var(--web2-border);\n\t\t\t\tcolor: var(--web2-muted);\n\t\t\t\tpadding: 0.2rem 0.6rem;\n\t\t\t\tborder-radius: 3px;\n\t\t\t\tfont-size: 0.85rem;\n\t\t\t\tletter-spacing: 0.02em;\n\t\t\t}\n\t\t\t.web2-link {\n\t\t\t\tcolor: var(--web2-blue);\n\t\t\t\ttext-decoration: none;\n\t\t\t\tfont-weight: 600;\n\t\t\t}\n\t\t\t.web2-link:hover {\n\t\t\t\ttext-decoration: underline;\n\t\t\t}\n\t\t\t.web2-button {\n\t\t\t\tbackground: var(--web2-blue);\n\t\t\t\tcolor: #fff;\n\t\t\t\tpadding: 0.45rem 0.9rem;\n\t\t\t\tborder-radius: 3px;\n\t\t\t\tborder: 1px solid #1482d0;\n\t\t\t\tbox-shadow: none;\n\t\t\t\tfont-weight: 600;\n\t\t\t\ttext-decoration: none;\n\t\t\t}\n\t\t\t.web2-button:hover {\n\t\t\t\tbackground: #1787d4;\n\t\t\t}\n\t\t\t.web2-list li {\n\t\t\t\tbackground: #ffffff;\n\t\t\t\tborder: 1px solid var(--web2-border);\n\t\t\t\tborder-radius: 3px;\n\t\t\t\tpadding: 0.75rem 1rem;\n\t\t\t\tbox-shadow: none;\n\t\t\t}\n\t\t</style></head>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}

View File

@@ -20,10 +20,10 @@ templ Index(info BuildInfo) {
<div class="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
<div>
<div class="web2-pill">vCTP Console</div>
<h1 class="mt-3 text-4xl font-bold">Build Intelligence Dashboard</h1>
<p class="mt-2 text-sm opacity-90">A glossy, snapshot-ready view of what is running.</p>
<h1 class="mt-3 text-4xl font-bold">Chargeback Intelligence Dashboard</h1>
<p class="mt-2 text-sm text-slate-600">Point in time snapshots of consumption.</p>
</div>
<div class="flex flex-wrap gap-3">
<div class="flex flex-wrap gap-4">
<a class="web2-button" href="/snapshots/hourly">Hourly Snapshots</a>
<a class="web2-button" href="/snapshots/daily">Daily Snapshots</a>
<a class="web2-button" href="/snapshots/monthly">Monthly Snapshots</a>

View File

@@ -47,14 +47,14 @@ func Index(info BuildInfo) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<body class=\"flex flex-col min-h-screen web2-bg\"><main class=\"flex-grow web2-shell space-y-8\"><section class=\"web2-header\"><div class=\"flex flex-col gap-4 md:flex-row md:items-center md:justify-between\"><div><div class=\"web2-pill\">vCTP Console</div><h1 class=\"mt-3 text-4xl font-bold\">Build Intelligence Dashboard</h1><p class=\"mt-2 text-sm opacity-90\">A glossy, snapshot-ready view of what is running.</p></div><div class=\"flex flex-wrap gap-3\"><a class=\"web2-button\" href=\"/snapshots/hourly\">Hourly Snapshots</a> <a class=\"web2-button\" href=\"/snapshots/daily\">Daily Snapshots</a> <a class=\"web2-button\" href=\"/snapshots/monthly\">Monthly Snapshots</a> <a class=\"web2-button\" href=\"/swagger/\">Swagger UI</a></div></div></section><section class=\"grid gap-6 md:grid-cols-3\"><div class=\"web2-card\"><p class=\"text-xs uppercase tracking-[0.2em] text-slate-400\">Build Time</p><p class=\"mt-3 text-xl font-semibold\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<body class=\"flex flex-col min-h-screen web2-bg\"><main class=\"flex-grow web2-shell space-y-8\"><section class=\"web2-header\"><div class=\"flex flex-col gap-4 md:flex-row md:items-center md:justify-between\"><div><div class=\"web2-pill\">vCTP Console</div><h1 class=\"mt-3 text-4xl font-bold\">Chargeback Intelligence Dashboard</h1><p class=\"mt-2 text-sm text-slate-600\">Point in time snapshots of consumption.</p></div><div class=\"flex flex-wrap gap-4\"><a class=\"web2-button\" href=\"/snapshots/hourly\">Hourly Snapshots</a> <a class=\"web2-button\" href=\"/snapshots/daily\">Daily Snapshots</a> <a class=\"web2-button\" href=\"/snapshots/monthly\">Monthly Snapshots</a> <a class=\"web2-button\" href=\"/swagger/\">Swagger UI</a></div></div></section><section class=\"grid gap-6 md:grid-cols-3\"><div class=\"web2-card\"><p class=\"text-xs uppercase tracking-[0.2em] text-slate-400\">Build Time</p><p class=\"mt-3 text-xl font-semibold\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(info.BuildTime)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 38, Col: 59}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 38, Col: 59}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
@@ -67,7 +67,7 @@ func Index(info BuildInfo) templ.Component {
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(info.SHA1Ver)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 42, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 42, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
@@ -80,7 +80,7 @@ func Index(info BuildInfo) templ.Component {
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(info.GoVersion)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 46, Col: 59}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 46, Col: 59}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
if templ_7745c5c3_Err != nil {

View File

@@ -32,15 +32,15 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {
<div>
<div class="web2-pill">Snapshot Library</div>
<h1 class="mt-3 text-4xl font-bold">{title}</h1>
<p class="mt-2 text-sm opacity-90">{subtitle}</p>
<p class="mt-2 text-sm text-slate-600">{subtitle}</p>
</div>
<a class="web2-button" href="/">Back to Dashboard</a>
</div>
</section>
<section class="web2-card">
<div class="flex items-center justify-between">
<h2 class="text-lg font-semibold">Available Exports</h2>
<div class="flex items-center justify-between gap-3">
<h2 class="text-lg font-semibold">Available Exports&nbsp;</h2>
<span class="text-xs uppercase tracking-[0.2em] text-slate-400">{len(entries)} files</span>
</div>
<ul class="mt-6 space-y-3 web2-list">

View File

@@ -140,33 +140,33 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 34, Col: 49}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</h1><p class=\"mt-2 text-sm opacity-90\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</h1><p class=\"mt-2 text-sm text-slate-600\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 35, Col: 51}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</p></div><a class=\"web2-button\" href=\"/\">Back to Dashboard</a></div></section><section class=\"web2-card\"><div class=\"flex items-center justify-between\"><h2 class=\"text-lg font-semibold\">Available Exports</h2><span class=\"text-xs uppercase tracking-[0.2em] text-slate-400\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</p></div><a class=\"web2-button\" href=\"/\">Back to Dashboard</a></div></section><section class=\"web2-card\"><div class=\"flex items-center justify-between gap-3\"><h2 class=\"text-lg font-semibold\">Available Exports&nbsp;</h2><span class=\"text-xs uppercase tracking-[0.2em] text-slate-400\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 44, Col: 83}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
@@ -184,7 +184,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var8 string
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 49, Col: 71}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
@@ -197,7 +197,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te
var templ_7745c5c3_Var9 templ.SafeURL
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 50, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {

View File

@@ -0,0 +1,14 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS snapshot_registry (
id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE snapshot_registry;
-- +goose StatementEnd

View File

@@ -0,0 +1,14 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS snapshot_registry (
id BIGSERIAL PRIMARY KEY,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
);
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TABLE snapshot_registry;
-- +goose StatementEnd

View File

@@ -66,3 +66,10 @@ CREATE TABLE IF NOT EXISTS inventory_history (
"PreviousResourcePool" TEXT,
"PreviousProvisionedDisk" REAL
);
CREATE TABLE IF NOT EXISTS snapshot_registry (
"id" INTEGER PRIMARY KEY AUTOINCREMENT,
"snapshot_type" TEXT NOT NULL,
"table_name" TEXT NOT NULL UNIQUE,
"snapshot_time" INTEGER NOT NULL
);

2
dist/dist.go vendored
View File

@@ -4,5 +4,5 @@ import (
"embed"
)
//go:embed all:assets
//go:embed all:assets favicon.ico favicon-16x16.png favicon-32x32.png
var AssetsDir embed.FS

BIN
dist/favicon-16x16.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 520 B

BIN
dist/favicon-32x32.png vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 KiB

BIN
dist/favicon.ico vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

View File

@@ -95,14 +95,33 @@ func beforeAll() {
func startApp() error {
port := getPort()
app = exec.Command("go", "run", "main.go")
settingsPath := "./test-settings.yml"
settingsBody := fmt.Sprintf(`settings:
log_level: "debug"
log_output: "text"
database_driver: "sqlite"
database_url: "./test-db.sqlite3"
bind_ip: "127.0.0.1"
bind_port: %d
bind_disable_tls: true
tls_cert_filename:
tls_key_filename:
vcenter_username: "test"
vcenter_password: "test"
vcenter_insecure: true
vcenter_event_polling_seconds: 60
vcenter_inventory_polling_seconds: 7200
vcenter_inventory_snapshot_seconds: 3600
vcenter_inventory_aggregate_seconds: 86400
hourly_snapshot_max_age_days: 1
daily_snapshot_max_age_months: 1
`, port)
if err := os.WriteFile("../"+settingsPath, []byte(settingsBody), 0o600); err != nil {
return err
}
app = exec.Command("go", "run", "main.go", "-settings", settingsPath)
app.Dir = "../"
app.Env = append(
os.Environ(),
"DB_URL=./test-db.sqlite3",
fmt.Sprintf("PORT=%d", port),
"LOG_LEVEL=DEBUG",
)
app.Env = os.Environ()
var err error
baseUrL, err = url.Parse(fmt.Sprintf("http://localhost:%d", port))
@@ -188,6 +207,9 @@ func afterAll() {
if err := os.Remove("../test-db.sqlite3"); err != nil {
log.Fatalf("could not remove test-db.sqlite3: %v", err)
}
if err := os.Remove("../test-settings.yml"); err != nil {
log.Fatalf("could not remove test-settings.yml: %v", err)
}
}
// beforeEach creates a new context and page for each test,

19
go.mod
View File

@@ -1,33 +1,42 @@
module vctp
go 1.25.0
go 1.25.5
require (
github.com/a-h/templ v0.3.977
github.com/go-co-op/gocron/v2 v2.19.0
github.com/jackc/pgx/v5 v5.8.0
github.com/jmoiron/sqlx v1.4.0
github.com/joho/godotenv v1.5.1
github.com/pressly/goose/v3 v3.26.0
github.com/swaggo/swag v1.16.6
github.com/vmware/govmomi v0.52.0
github.com/xuri/excelize/v2 v2.10.0
gopkg.in/yaml.v2 v2.4.0
modernc.org/sqlite v1.43.0
modernc.org/sqlite v1.44.0
)
require (
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
github.com/go-openapi/spec v0.20.4 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.6 // indirect
github.com/richardlehane/msoleps v1.0.5 // indirect
github.com/richardlehane/msoleps v1.0.6 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/tiendc/go-deepcopy v1.7.2 // indirect
@@ -36,10 +45,12 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.47.0 // indirect
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
golang.org/x/mod v0.32.0 // indirect
golang.org/x/net v0.49.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
golang.org/x/tools v0.41.0 // indirect
modernc.org/libc v1.67.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect

47
go.sum
View File

@@ -1,7 +1,14 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -9,6 +16,16 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-co-op/gocron/v2 v2.19.0 h1:OKf2y6LXPs/BgBI2fl8PxUpNAI1DA9Mg+hSeGOS38OU=
github.com/go-co-op/gocron/v2 v2.19.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
@@ -30,16 +47,23 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
@@ -48,6 +72,7 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM=
@@ -58,6 +83,8 @@ github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq
github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
github.com/richardlehane/msoleps v1.0.5 h1:kNlmACZuwC8ZWPLoJtD+HtZOsKJgYn7gXgUIcRB7dbo=
github.com/richardlehane/msoleps v1.0.5/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg=
github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
@@ -66,9 +93,12 @@ github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI=
github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg=
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw=
@@ -91,23 +121,34 @@ golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
@@ -134,6 +175,8 @@ modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.43.0 h1:8YqiFx3G1VhHTXO2Q00bl1Wz9KhS9Q5okwfp9Y97VnA=
modernc.org/sqlite v1.43.0/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc=
modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

View File

@@ -14,6 +14,12 @@ import (
"github.com/xuri/excelize/v2"
)
type SnapshotRecord struct {
TableName string
SnapshotTime time.Time
SnapshotType string
}
func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) {
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
@@ -59,26 +65,139 @@ ORDER BY tablename DESC
return tables, rows.Err()
}
func FormatSnapshotLabel(prefix string, tableName string) (string, bool) {
if !strings.HasPrefix(tableName, prefix) {
return "", false
}
suffix := strings.TrimPrefix(tableName, prefix)
switch prefix {
case "inventory_daily_":
if t, err := time.Parse("20060102", suffix); err == nil {
return t.Format("2006-01-02"), true
}
case "inventory_daily_summary_":
if t, err := time.Parse("20060102", suffix); err == nil {
return t.Format("2006-01-02"), true
}
case "inventory_monthly_summary_":
if t, err := time.Parse("200601", suffix); err == nil {
return t.Format("2006-01"), true
func EnsureSnapshotRegistry(ctx context.Context, database db.Database) error {
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
switch driver {
case "sqlite":
_, err := dbConn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS snapshot_registry (
id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
)
`)
return err
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS snapshot_registry (
id BIGSERIAL PRIMARY KEY,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
)
`)
return err
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
}
}
return "", false
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty")
}
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
switch driver {
case "sqlite":
_, err := dbConn.ExecContext(ctx, `
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES (?, ?, ?)
`, snapshotType, tableName, snapshotTime.Unix())
return err
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES ($1, $2, $3)
ON CONFLICT (table_name) DO NOTHING
`, snapshotType, tableName, snapshotTime.Unix())
return err
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
}
}
func DeleteSnapshotRecord(ctx context.Context, database db.Database, tableName string) error {
if tableName == "" {
return nil
}
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
switch driver {
case "sqlite":
_, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry WHERE table_name = ?`, tableName)
return err
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry WHERE table_name = $1`, tableName)
return err
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
}
}
func ListSnapshots(ctx context.Context, database db.Database, snapshotType string) ([]SnapshotRecord, error) {
dbConn := database.DB()
driver := strings.ToLower(dbConn.DriverName())
var rows *sqlx.Rows
var err error
switch driver {
case "sqlite":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
FROM snapshot_registry
WHERE snapshot_type = ?
ORDER BY snapshot_time DESC, table_name DESC
`, snapshotType)
case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
FROM snapshot_registry
WHERE snapshot_type = $1
ORDER BY snapshot_time DESC, table_name DESC
`, snapshotType)
default:
return nil, fmt.Errorf("unsupported driver for listing snapshots: %s", driver)
}
if err != nil {
return nil, err
}
defer rows.Close()
records := make([]SnapshotRecord, 0)
for rows.Next() {
var (
tableName string
snapshotTime int64
recordType string
)
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
return nil, err
}
records = append(records, SnapshotRecord{
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
})
}
return records, rows.Err()
}
func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName string) string {
switch snapshotType {
case "hourly":
return snapshotTime.Format("2006-01-02 15:00")
case "daily":
return snapshotTime.Format("2006-01-02")
case "monthly":
return snapshotTime.Format("2006-01")
default:
return tableName
}
}
func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string) ([]byte, error) {

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"os"
"path/filepath"
"vctp/internal/utils"
"gopkg.in/yaml.v2"
@@ -19,6 +20,25 @@ type Settings struct {
// SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties
type SettingsYML struct {
Settings struct {
LogLevel string `yaml:"log_level"`
LogOutput string `yaml:"log_output"`
DatabaseDriver string `yaml:"database_driver"`
DatabaseURL string `yaml:"database_url"`
BindIP string `yaml:"bind_ip"`
BindPort int `yaml:"bind_port"`
BindDisableTLS bool `yaml:"bind_disable_tls"`
TLSCertFilename string `yaml:"tls_cert_filename"`
TLSKeyFilename string `yaml:"tls_key_filename"`
VcenterUsername string `yaml:"vcenter_username"`
VcenterPassword string `yaml:"vcenter_password"`
VcenterInsecure bool `yaml:"vcenter_insecure"`
VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"`
VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"`
VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"`
VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"`
HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"`
DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"`
SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"`
TenantsToFilter []string `yaml:"tenants_to_filter"`
NodeChargeClusters []string `yaml:"node_charge_clusters"`
SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"`
@@ -65,3 +85,49 @@ func (s *Settings) ReadYMLSettings() error {
return nil
}
func (s *Settings) WriteYMLSettings() error {
if s.Values == nil {
return errors.New("settings are not loaded")
}
if len(s.SettingsPath) == 0 {
return errors.New("settings file path not specified")
}
data, err := yaml.Marshal(s.Values)
if err != nil {
return fmt.Errorf("unable to encode settings file: %w", err)
}
mode := os.FileMode(0o644)
if info, err := os.Stat(s.SettingsPath); err == nil {
mode = info.Mode().Perm()
}
dir := filepath.Dir(s.SettingsPath)
tmp, err := os.CreateTemp(dir, "vctp-settings-*.yml")
if err != nil {
return fmt.Errorf("unable to create temp settings file: %w", err)
}
tmpName := tmp.Name()
defer func() {
_ = os.Remove(tmpName)
}()
if _, err := tmp.Write(data); err != nil {
_ = tmp.Close()
return fmt.Errorf("unable to write temp settings file: %w", err)
}
if err := tmp.Chmod(mode); err != nil {
_ = tmp.Close()
return fmt.Errorf("unable to set temp settings permissions: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("unable to close temp settings file: %w", err)
}
if err := os.Rename(tmpName, s.SettingsPath); err != nil {
return fmt.Errorf("unable to replace settings file: %w", err)
}
return nil
}

View File

@@ -5,8 +5,6 @@ import (
"database/sql"
"fmt"
"log/slog"
"os"
"strconv"
"strings"
"time"
"vctp/db/queries"
@@ -33,8 +31,8 @@ type inventorySnapshotRow struct {
Cluster sql.NullString
Folder sql.NullString
ProvisionedDisk sql.NullFloat64
InitialVcpus sql.NullInt64
InitialRam sql.NullInt64
VcpuCount sql.NullInt64
RamGB sql.NullInt64
IsTemplate string
PoweredOn string
SrmPlaceholder string
@@ -46,7 +44,7 @@ type inventorySnapshotRow struct {
// RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table.
func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) error {
startTime := time.Now()
tableName, err := dailyInventoryTableName(startTime)
tableName, err := hourlyInventoryTableName(startTime)
if err != nil {
return err
}
@@ -55,6 +53,12 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil {
return err
}
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil {
c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName)
}
// reload settings in case vcenter list has changed
c.Settings.ReadYMLSettings()
@@ -62,7 +66,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
for _, url := range c.Settings.Values.Settings.VcenterAddresses {
c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url)
vc := vcenter.New(c.Logger, c.VcCreds)
vc.Login(url)
if err := vc.Login(url); err != nil {
c.Logger.Error("unable to connect to vcenter for hourly snapshot", "error", err, "url", url)
continue
}
vcVms, err := vc.GetAllVmReferences()
if err != nil {
@@ -70,6 +77,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
vc.Logout()
continue
}
canDetectMissing := len(vcVms) > 0
if !canDetectMissing {
c.Logger.Warn("no VMs returned from vcenter; skipping missing VM detection", "url", url)
}
inventoryRows, err := c.Database.Queries().GetInventoryByVcenter(ctx, url)
if err != nil {
@@ -116,8 +127,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
presentSnapshots[vm.Reference().Value] = row
totals.VmCount++
totals.VcpuTotal += nullInt64ToInt(row.InitialVcpus)
totals.RamTotal += nullInt64ToInt(row.InitialRam)
totals.VcpuTotal += nullInt64ToInt(row.VcpuCount)
totals.RamTotal += nullInt64ToInt(row.RamGB)
totals.DiskTotal += nullFloat64ToFloat(row.ProvisionedDisk)
}
@@ -127,7 +138,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
}
}
if !canDetectMissing {
vc.Logout()
continue
}
for _, inv := range inventoryRows {
if strings.HasPrefix(inv.Name, "vCLS-") {
continue
}
vmID := inv.VmId.String
if vmID != "" {
if _, ok := presentSnapshots[vmID]; ok {
@@ -137,6 +156,17 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
row := snapshotFromInventory(inv, startTime)
row.IsPresent = "FALSE"
if !row.DeletionTime.Valid {
deletionTime := startTime.Unix()
row.DeletionTime = sql.NullInt64{Int64: deletionTime, Valid: true}
if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{
DeletionTime: row.DeletionTime,
VmId: inv.VmId,
DatacenterName: inv.Datacenter,
}); err != nil {
c.Logger.Warn("failed to mark inventory record deleted", "error", err, "vm_id", row.VmId.String)
}
}
if err := insertDailyInventoryRow(ctx, dbConn, tableName, row); err != nil {
c.Logger.Error("failed to insert missing VM snapshot", "error", err, "vm_id", row.VmId.String)
}
@@ -148,7 +178,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
"vcenter", url,
"vm_count", totals.VmCount,
"vcpu_total", totals.VcpuTotal,
"ram_total_mb", totals.RamTotal,
"ram_total_gb", totals.RamTotal,
"disk_total_gb", totals.DiskTotal,
)
}
@@ -160,7 +190,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo
// RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table.
func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error {
targetTime := time.Now().Add(-time.Minute)
sourceTable, err := dailyInventoryTableName(targetTime)
sourceTable, err := hourlyInventoryTableName(targetTime)
if err != nil {
return err
}
@@ -173,6 +203,9 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo
if err := ensureDailySummaryTable(ctx, dbConn, summaryTable); err != nil {
return err
}
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
return err
}
currentTotals, err := snapshotTotalsForTable(ctx, dbConn, sourceTable)
if err != nil {
@@ -182,12 +215,12 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo
"table", sourceTable,
"vm_count", currentTotals.VmCount,
"vcpu_total", currentTotals.VcpuTotal,
"ram_total_mb", currentTotals.RamTotal,
"ram_total_gb", currentTotals.RamTotal,
"disk_total_gb", currentTotals.DiskTotal,
)
}
prevTable, _ := dailyInventoryTableName(targetTime.AddDate(0, 0, -1))
prevTable, _ := hourlyInventoryTableName(targetTime.AddDate(0, 0, -1))
if prevTable != "" && tableExists(ctx, dbConn, prevTable) {
prevTotals, err := snapshotTotalsForTable(ctx, dbConn, prevTable)
if err != nil {
@@ -198,7 +231,7 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo
"previous_table", prevTable,
"vm_delta", currentTotals.VmCount-prevTotals.VmCount,
"vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal,
"ram_delta_mb", currentTotals.RamTotal-prevTotals.RamTotal,
"ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal,
"disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal,
)
}
@@ -207,19 +240,19 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"SamplesPresent", "AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct"
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB",
AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk",
AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct",
@@ -232,14 +265,17 @@ SELECT
FROM %s
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, summaryTable, sourceTable)
if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil {
c.Logger.Error("failed to aggregate daily inventory", "error", err, "source_table", sourceTable)
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, targetTime); err != nil {
c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable)
}
c.Logger.Debug("Finished daily inventory aggregation", "source_table", sourceTable, "summary_table", summaryTable)
return nil
@@ -251,7 +287,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.
firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location())
targetMonth := firstOfThisMonth.AddDate(0, -1, 0)
monthPrefix := fmt.Sprintf("inventory_daily_%s", targetMonth.Format("200601"))
monthPrefix := fmt.Sprintf("inventory_hourly_%s", targetMonth.Format("200601"))
dailyTables, err := report.ListTablesByPrefix(ctx, c.Database, monthPrefix)
if err != nil {
return err
@@ -269,11 +305,14 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.
if err := ensureMonthlySummaryTable(ctx, dbConn, monthlyTable); err != nil {
return err
}
if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil {
return err
}
unionQuery := buildUnionQuery(dailyTables, []string{
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
`"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
`"ProvisionedDisk"`, `"InitialVcpus"`, `"InitialRam"`, `"IsTemplate"`, `"PoweredOn"`,
`"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`,
`"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`,
})
if strings.TrimSpace(unionQuery) == "" {
@@ -288,7 +327,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.
"month", targetMonth.Format("2006-01"),
"vm_count", monthlyTotals.VmCount,
"vcpu_total", monthlyTotals.VcpuTotal,
"ram_total_mb", monthlyTotals.RamTotal,
"ram_total_gb", monthlyTotals.RamTotal,
"disk_total_gb", monthlyTotals.DiskTotal,
)
}
@@ -296,18 +335,18 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog.
insertQuery := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
"AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent",
"PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct"
)
SELECT
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
AVG(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus",
AVG(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam",
AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid",
AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount",
AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB",
AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk",
AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent",
100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END)
/ NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct",
@@ -322,14 +361,17 @@ FROM (
) snapshots
GROUP BY
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid";
`, monthlyTable, unionQuery)
if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil {
c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01"))
return err
}
if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil {
c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable)
}
c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable)
return nil
@@ -338,15 +380,15 @@ GROUP BY
// RunSnapshotCleanup drops hourly and daily snapshot tables older than retention.
func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error {
now := time.Now()
hourlyMaxDays := getEnvInt("HOURLY_SNAPSHOT_MAX_AGE_DAYS", 60)
dailyMaxMonths := getEnvInt("DAILY_SNAPSHOT_MAX_AGE_MONTHS", 12)
hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60)
dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12)
hourlyCutoff := now.AddDate(0, 0, -hourlyMaxDays)
dailyCutoff := now.AddDate(0, -dailyMaxMonths, 0)
dbConn := c.Database.DB()
hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_daily_")
hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_hourly_")
if err != nil {
return err
}
@@ -356,7 +398,7 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger)
if strings.HasPrefix(table, "inventory_daily_summary_") {
continue
}
tableDate, ok := parseSnapshotDate(table, "inventory_daily_", "20060102")
tableDate, ok := parseSnapshotDate(table, "inventory_hourly_", "2006010215")
if !ok {
continue
}
@@ -365,6 +407,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger)
c.Logger.Error("failed to drop hourly snapshot table", "error", err, "table", table)
} else {
removedHourly++
if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil {
c.Logger.Warn("failed to remove hourly snapshot registry entry", "error", err, "table", table)
}
}
}
}
@@ -384,6 +429,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger)
c.Logger.Error("failed to drop daily snapshot table", "error", err, "table", table)
} else {
removedDaily++
if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil {
c.Logger.Warn("failed to remove daily snapshot registry entry", "error", err, "table", table)
}
}
}
}
@@ -397,8 +445,8 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger)
return nil
}
func dailyInventoryTableName(t time.Time) (string, error) {
return safeTableName(fmt.Sprintf("inventory_daily_%s", t.Format("20060102")))
func hourlyInventoryTableName(t time.Time) (string, error) {
return safeTableName(fmt.Sprintf("inventory_hourly_%s", t.Format("2006010215")))
}
func dailySummaryTableName(t time.Time) (string, error) {
@@ -435,8 +483,8 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"Cluster" TEXT,
"Folder" TEXT,
"ProvisionedDisk" REAL,
"InitialVcpus" BIGINT,
"InitialRam" BIGINT,
"VcpuCount" BIGINT,
"RamGB" BIGINT,
"IsTemplate" TEXT,
"PoweredOn" TEXT,
"SrmPlaceholder" TEXT,
@@ -445,10 +493,16 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"IsPresent" TEXT NOT NULL
);`, tableName)
_, err := dbConn.ExecContext(ctx, ddl)
if _, err := dbConn.ExecContext(ctx, ddl); err != nil {
return err
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "VcpuCount", Type: "BIGINT"},
{Name: "RamGB", Type: "BIGINT"},
})
}
func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error {
ddl := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
"InventoryId" BIGINT,
@@ -465,16 +519,16 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
"Cluster" TEXT,
"Folder" TEXT,
"ProvisionedDisk" REAL,
"InitialVcpus" BIGINT,
"InitialRam" BIGINT,
"VcpuCount" BIGINT,
"RamGB" BIGINT,
"IsTemplate" TEXT,
"PoweredOn" TEXT,
"SrmPlaceholder" TEXT,
"VmUuid" TEXT,
"SamplesPresent" BIGINT NOT NULL,
"AvgVcpus" REAL,
"AvgRam" REAL,
"AvgDisk" REAL,
"AvgVcpuCount" REAL,
"AvgRamGB" REAL,
"AvgProvisionedDisk" REAL,
"AvgIsPresent" REAL,
"PoolTinPct" REAL,
"PoolBronzePct" REAL,
@@ -487,9 +541,9 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgVcpus", Type: "REAL"},
{Name: "AvgRam", Type: "REAL"},
{Name: "AvgDisk", Type: "REAL"},
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
{Name: "AvgProvisionedDisk", Type: "REAL"},
{Name: "AvgIsPresent", Type: "REAL"},
{Name: "PoolTinPct", Type: "REAL"},
{Name: "PoolBronzePct", Type: "REAL"},
@@ -514,15 +568,15 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
"Cluster" TEXT,
"Folder" TEXT,
"ProvisionedDisk" REAL,
"InitialVcpus" BIGINT,
"InitialRam" BIGINT,
"VcpuCount" BIGINT,
"RamGB" BIGINT,
"IsTemplate" TEXT,
"PoweredOn" TEXT,
"SrmPlaceholder" TEXT,
"VmUuid" TEXT,
"AvgVcpus" REAL,
"AvgRam" REAL,
"AvgDisk" REAL,
"AvgVcpuCount" REAL,
"AvgRamGB" REAL,
"AvgProvisionedDisk" REAL,
"AvgIsPresent" REAL,
"PoolTinPct" REAL,
"PoolBronzePct" REAL,
@@ -535,7 +589,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s
}
return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{
{Name: "AvgDisk", Type: "REAL"},
{Name: "AvgVcpuCount", Type: "REAL"},
{Name: "AvgRamGB", Type: "REAL"},
{Name: "AvgProvisionedDisk", Type: "REAL"},
{Name: "AvgIsPresent", Type: "REAL"},
{Name: "PoolTinPct", Type: "REAL"},
{Name: "PoolBronzePct", Type: "REAL"},
{Name: "PoolSilverPct", Type: "REAL"},
@@ -622,8 +679,8 @@ func snapshotTotalsForTable(ctx context.Context, dbConn *sqlx.DB, table string)
query := fmt.Sprintf(`
SELECT
COUNT(DISTINCT "VmId") AS vm_count,
COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total,
COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total,
COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total,
COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total,
COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total
FROM %s
WHERE "IsPresent" = 'TRUE'
@@ -640,8 +697,8 @@ func snapshotTotalsForUnion(ctx context.Context, dbConn *sqlx.DB, unionQuery str
query := fmt.Sprintf(`
SELECT
COUNT(DISTINCT "VmId") AS vm_count,
COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total,
COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total,
COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total,
COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total,
COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total
FROM (
%s
@@ -694,13 +751,8 @@ func nullFloat64ToFloat(value sql.NullFloat64) float64 {
return 0
}
func getEnvInt(key string, fallback int) int {
raw := strings.TrimSpace(os.Getenv(key))
if raw == "" {
return fallback
}
value, err := strconv.Atoi(raw)
if err != nil || value < 0 {
func intWithDefault(value int, fallback int) int {
if value <= 0 {
return fallback
}
return value
@@ -731,8 +783,8 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
if !vmObject.Config.CreateDate.IsZero() {
row.CreationTime = sql.NullInt64{Int64: vmObject.Config.CreateDate.Unix(), Valid: true}
}
row.InitialVcpus = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0}
row.InitialRam = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB), Valid: vmObject.Config.Hardware.MemoryMB > 0}
row.VcpuCount = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0}
row.RamGB = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB) / 1024, Valid: vmObject.Config.Hardware.MemoryMB > 0}
totalDiskBytes := int64(0)
for _, device := range vmObject.Config.Hardware.Device {
@@ -774,11 +826,11 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi
if !row.ProvisionedDisk.Valid {
row.ProvisionedDisk = inv.ProvisionedDisk
}
if !row.InitialVcpus.Valid {
row.InitialVcpus = inv.InitialVcpus
if !row.VcpuCount.Valid {
row.VcpuCount = inv.InitialVcpus
}
if !row.InitialRam.Valid {
row.InitialRam = inv.InitialRam
if !row.RamGB.Valid && inv.InitialRam.Valid {
row.RamGB = sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Int64 > 0}
}
if row.IsTemplate == "" {
row.IsTemplate = boolStringFromInterface(inv.IsTemplate)
@@ -837,8 +889,8 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
Cluster: inv.Cluster,
Folder: inv.Folder,
ProvisionedDisk: inv.ProvisionedDisk,
InitialVcpus: inv.InitialVcpus,
InitialRam: inv.InitialRam,
VcpuCount: inv.InitialVcpus,
RamGB: sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Valid && inv.InitialRam.Int64 > 0},
IsTemplate: boolStringFromInterface(inv.IsTemplate),
PoweredOn: boolStringFromInterface(inv.PoweredOn),
SrmPlaceholder: boolStringFromInterface(inv.SrmPlaceholder),
@@ -851,8 +903,8 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str
query := fmt.Sprintf(`
INSERT INTO %s (
"InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime",
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus",
"InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
"ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount",
"RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent"
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
`, tableName)
@@ -874,8 +926,8 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
row.Cluster,
row.Folder,
row.ProvisionedDisk,
row.InitialVcpus,
row.InitialRam,
row.VcpuCount,
row.RamGB,
row.IsTemplate,
row.PoweredOn,
row.SrmPlaceholder,

View File

@@ -262,6 +262,11 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce
return errors.New("can't process empty vm object")
}
if strings.HasPrefix(vmObject.Name, "vCLS-") {
c.Logger.Debug("Skipping internal vCLS VM", "vm_name", vmObject.Name)
return nil
}
c.Logger.Debug("found VM")
/*

View File

@@ -83,6 +83,14 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error {
continue
}
if strings.HasPrefix(vmObject.Name, "vCLS-") {
c.Logger.Info("Skipping internal vCLS VM event", "vm_name", vmObject.Name)
if err := c.Database.Queries().UpdateEventsProcessed(ctx, evt.Eid); err != nil {
c.Logger.Error("Unable to mark vCLS event as processed", "event_id", evt.Eid, "error", err)
}
continue
}
//c.Logger.Debug("found VM")
srmPlaceholder = "FALSE" // Default assumption
//prettyPrint(vmObject)

View File

@@ -6,7 +6,6 @@ import (
"log"
"log/slog"
"net/url"
"os"
"path"
"strings"
@@ -30,6 +29,7 @@ type Vcenter struct {
type VcenterLogin struct {
Username string
Password string
Insecure bool
}
type VmProperties struct {
@@ -51,13 +51,6 @@ func New(logger *slog.Logger, creds *VcenterLogin) *Vcenter {
}
func (v *Vcenter) Login(vUrl string) error {
var insecure bool
// TODO - fix this
insecureString := os.Getenv("VCENTER_INSECURE")
//username := os.Getenv("VCENTER_USERNAME")
//password := os.Getenv("VCENTER_PASSWORD")
// Connect to vCenter
u, err := soap.ParseURL(vUrl)
if err != nil {
@@ -74,11 +67,7 @@ func (v *Vcenter) Login(vUrl string) error {
}
*/
if insecureString == "true" {
insecure = true
}
c, err := govmomi.NewClient(v.ctx, u, insecure)
c, err := govmomi.NewClient(v.ctx, u, v.credentials.Insecure)
if err != nil {
v.Logger.Error("Unable to connect to vCenter", "error", err)
return fmt.Errorf("unable to connect to vCenter : %s", err)

View File

@@ -65,10 +65,9 @@ func ToLevel(level string) Level {
}
}
// GetLevel returns the log level from the environment variable.
// GetLevel returns the default log level.
func GetLevel() Level {
level := os.Getenv("LOG_LEVEL")
return ToLevel(level)
return LevelInfo
}
// Output represents the log output.
@@ -93,8 +92,7 @@ func ToOutput(output string) Output {
}
}
// GetOutput returns the log output from the environment variable.
// GetOutput returns the default log output.
func GetOutput() Output {
output := os.Getenv("LOG_OUTPUT")
return ToOutput(output)
return OutputText
}

155
main.go
View File

@@ -2,8 +2,8 @@ package main
import (
"context"
"flag"
"fmt"
"log/slog"
"os"
"runtime"
"strings"
@@ -19,7 +19,6 @@ import (
"vctp/server/router"
"github.com/go-co-op/gocron/v2"
"github.com/joho/godotenv"
)
var (
@@ -34,22 +33,29 @@ var (
)
func main() {
// Load data from environment file
envFilename := utils.GetFilePath(".env")
err := godotenv.Load(envFilename)
if err != nil {
panic("Error loading .env file")
}
settingsPath := flag.String("settings", "/etc/dtms/vctp.yml", "Path to settings YAML")
flag.Parse()
logger := log.New(
log.GetLevel(),
log.GetOutput(),
)
bootstrapLogger := log.New(log.LevelInfo, log.OutputText)
ctx, cancel := context.WithCancel(context.Background())
// Load settings from yaml
s := settings.New(bootstrapLogger, *settingsPath)
err := s.ReadYMLSettings()
if err != nil {
bootstrapLogger.Error("failed to open yaml settings file", "error", err, "filename", *settingsPath)
os.Exit(1)
}
logger := log.New(
log.ToLevel(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogLevel))),
log.ToOutput(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogOutput))),
)
s.Logger = logger
// Configure database
dbDriver := os.Getenv("DB_DRIVER")
dbDriver := strings.TrimSpace(s.Values.Settings.DatabaseDriver)
if dbDriver == "" {
dbDriver = "sqlite"
}
@@ -57,12 +63,12 @@ func main() {
if normalizedDriver == "" || normalizedDriver == "sqlite3" {
normalizedDriver = "sqlite"
}
dbURL := os.Getenv("DB_URL")
dbURL := strings.TrimSpace(s.Values.Settings.DatabaseURL)
if dbURL == "" && normalizedDriver == "sqlite" {
dbURL = utils.GetFilePath("db.sqlite3")
}
database, err := db.New(logger, db.Config{Driver: dbDriver, DSN: dbURL})
database, err := db.New(logger, db.Config{Driver: normalizedDriver, DSN: dbURL})
if err != nil {
logger.Error("Failed to create database", "error", err)
os.Exit(1)
@@ -70,56 +76,36 @@ func main() {
defer database.Close()
//defer database.DB().Close()
if err = db.Migrate(database, dbDriver); err != nil {
if err = db.Migrate(database, normalizedDriver); err != nil {
logger.Error("failed to migrate database", "error", err)
os.Exit(1)
}
// Load settings from yaml
settingsFile := os.Getenv("SETTINGS_FILE")
if settingsFile == "" {
settingsFile = "settings.yaml"
}
// TODO - how to pass this to the other packages that will need this info?
s := settings.New(logger, settingsFile)
err = s.ReadYMLSettings()
//s, err := settings.ReadYMLSettings(logger, settingsFile)
if err != nil {
logger.Error("failed to open yaml settings file", "error", err, "filename", settingsFile)
//os.Exit(1)
} else {
logger.Debug("Loaded yaml settings", "contents", s)
}
// Determine bind IP
bindIP := os.Getenv("BIND_IP")
bindIP := strings.TrimSpace(s.Values.Settings.BindIP)
if bindIP == "" {
bindIP = utils.GetOutboundIP().String()
}
// Determine bind port
bindPort := os.Getenv("BIND_PORT")
if bindPort == "" {
bindPort = "9443"
bindPort := s.Values.Settings.BindPort
if bindPort == 0 {
bindPort = 9443
}
bindAddress := fmt.Sprint(bindIP, ":", bindPort)
//logger.Info("Will listen on address", "ip", bindIP, "port", bindPort)
// Determine bind disable TLS
bindDisableTlsEnv := os.Getenv("BIND_DISABLE_TLS")
if bindDisableTlsEnv == "true" {
bindDisableTls = true
}
bindDisableTls = s.Values.Settings.BindDisableTLS
// Get file names for TLS cert/key
tlsCertFilename := os.Getenv("TLS_CERT_FILE")
tlsCertFilename := strings.TrimSpace(s.Values.Settings.TLSCertFilename)
if tlsCertFilename != "" {
tlsCertFilename = utils.GetFilePath(tlsCertFilename)
} else {
tlsCertFilename = "./cert.pem"
}
tlsKeyFilename := os.Getenv("TLS_KEY_FILE")
tlsKeyFilename := strings.TrimSpace(s.Values.Settings.TLSKeyFilename)
if tlsKeyFilename != "" {
tlsKeyFilename = utils.GetFilePath(tlsKeyFilename)
} else {
@@ -132,9 +118,9 @@ func main() {
utils.GenerateCerts(tlsCertFilename, tlsKeyFilename)
}
// Load vcenter credentials from .env
// Load vcenter credentials from serttings, decrypt if required
a := secrets.New(logger, encryptionKey)
vcEp := os.Getenv("VCENTER_PASSWORD")
vcEp := strings.TrimSpace(s.Values.Settings.VcenterPassword)
if len(vcEp) == 0 {
logger.Error("No vcenter password configured")
os.Exit(1)
@@ -143,13 +129,27 @@ func main() {
if err != nil {
logger.Error("failed to decrypt vcenter credentials. Assuming un-encrypted", "error", err)
vcPass = []byte(vcEp)
if cipherText, encErr := a.Encrypt([]byte(vcEp)); encErr != nil {
logger.Warn("failed to encrypt vcenter credentials", "error", encErr)
} else {
s.Values.Settings.VcenterPassword = cipherText
if err := s.WriteYMLSettings(); err != nil {
logger.Warn("failed to update settings with encrypted vcenter password", "error", err)
} else {
logger.Info("encrypted vcenter password stored in settings file")
}
}
//os.Exit(1)
}
creds := vcenter.VcenterLogin{
//insecureString := os.Getenv("VCENTER_INSECURE")
Username: os.Getenv("VCENTER_USERNAME"),
Username: strings.TrimSpace(s.Values.Settings.VcenterUsername),
Password: string(vcPass),
Insecure: s.Values.Settings.VcenterInsecure,
}
if creds.Username == "" {
logger.Error("No vcenter username configured")
os.Exit(1)
}
// Prepare the task scheduler
@@ -167,54 +167,19 @@ func main() {
VcCreds: &creds,
}
cronFrequencyString := os.Getenv("VCENTER_EVENT_POLLING_SECONDS")
if cronFrequencyString != "" {
cronFrequency, err = time.ParseDuration(cronFrequencyString)
if err != nil {
slog.Error("Can't convert VCENTER_EVENT_POLLING_SECONDS value to time duration. Defaulting to 60s", "value", cronFrequencyString, "error", err)
cronFrequency = time.Second * 60
}
} else {
cronFrequency = time.Second * 60
}
cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60)
logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency)
cronInventoryFrequencyString := os.Getenv("VCENTER_INVENTORY_POLLING_SECONDS")
if cronInventoryFrequencyString != "" {
cronInvFrequency, err = time.ParseDuration(cronInventoryFrequencyString)
if err != nil {
slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 7200", "value", cronInventoryFrequencyString, "error", err)
cronInvFrequency = time.Second * 7200
}
} else {
cronInvFrequency = time.Second * 7200
}
cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200)
logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency)
cronSnapshotFrequencyString := os.Getenv("VCENTER_INVENTORY_SNAPSHOT_SECONDS")
if cronSnapshotFrequencyString != "" {
cronSnapshotFrequency, err = time.ParseDuration(cronSnapshotFrequencyString)
if err != nil {
slog.Error("Can't convert VCENTER_INVENTORY_SNAPSHOT_SECONDS value to time duration. Defaulting to 3600", "value", cronSnapshotFrequencyString, "error", err)
cronSnapshotFrequency = time.Hour
}
} else {
cronSnapshotFrequency = time.Hour
}
cronSnapshotFrequency = durationFromSeconds(s.Values.Settings.VcenterInventorySnapshotSeconds, 3600)
logger.Debug("Setting VM inventory snapshot cronjob frequency to", "frequency", cronSnapshotFrequency)
cronAggregateFrequencyString := os.Getenv("VCENTER_INVENTORY_AGGREGATE_SECONDS")
if cronAggregateFrequencyString != "" {
cronAggregateFrequency, err = time.ParseDuration(cronAggregateFrequencyString)
if err != nil {
slog.Error("Can't convert VCENTER_INVENTORY_AGGREGATE_SECONDS value to time duration. Defaulting to 86400", "value", cronAggregateFrequencyString, "error", err)
cronAggregateFrequency = time.Hour * 24
}
} else {
cronAggregateFrequency = time.Hour * 24
}
cronAggregateFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryAggregateSeconds, 86400)
logger.Debug("Setting VM inventory aggregation cronjob frequency to", "frequency", cronAggregateFrequency)
/*
// start background processing for events stored in events table
startsAt := time.Now().Add(time.Second * 10)
job, err := c.NewJob(
@@ -229,8 +194,10 @@ func main() {
os.Exit(1)
}
logger.Debug("Created event processing cron job", "job", job.ID(), "starting_at", startsAt)
*/
// start background checks of vcenter inventory
/*
startsAt2 := time.Now().Add(cronInvFrequency)
job2, err := c.NewJob(
gocron.DurationJob(cronInvFrequency),
@@ -244,6 +211,7 @@ func main() {
os.Exit(1)
}
logger.Debug("Created vcenter inventory cron job", "job", job2.ID(), "starting_at", startsAt2)
*/
startsAt3 := time.Now().Add(cronSnapshotFrequency)
if cronSnapshotFrequency == time.Hour {
@@ -292,8 +260,12 @@ func main() {
}
logger.Debug("Created vcenter monthly aggregation cron job", "job", job5.ID())
snapshotCleanupCron := strings.TrimSpace(s.Values.Settings.SnapshotCleanupCron)
if snapshotCleanupCron == "" {
snapshotCleanupCron = "30 2 * * *"
}
job6, err := c.NewJob(
gocron.CronJob("0 30 2 * *", false),
gocron.CronJob(snapshotCleanupCron, false),
gocron.NewTask(func() {
ct.RunSnapshotCleanup(ctx, logger)
}), gocron.WithSingletonMode(gocron.LimitModeReschedule),
@@ -325,3 +297,10 @@ func main() {
os.Exit(0)
}
func durationFromSeconds(value int, fallback int) time.Duration {
if value <= 0 {
return time.Second * time.Duration(fallback)
}
return time.Second * time.Duration(value)
}

View File

@@ -10,8 +10,13 @@ buildtime=$(date +%Y-%m-%dT%T%z)
#Extract the version from yml
package_version=$(grep 'version:' "$package_name.yml" | awk '{print $2}' | tr -d '"' | sed 's/^v//')
#platforms=("linux/amd64" "darwin/amd64")
host_os=$(uname -s | tr '[:upper:]' '[:lower:]')
host_arch=$(uname -m)
platforms=("linux/amd64")
if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then
platforms+=("darwin/amd64")
fi
#platforms=("linux/amd64")
echo Building::
echo - Version $package_version
@@ -40,6 +45,4 @@ do
#sha256sum build/${output_name}.gz > build/${output_name}_checksum.txt
done
nfpm package --config $package_name.yml --packager rpm --target build/
ls -lah build

View File

@@ -4,7 +4,7 @@ set -euo pipefail
# Usage: ./update-swagger-ui.sh [version]
# Example: ./update-swagger-ui.sh v5.17.14
# If no version is provided, defaults below is used.
VERSION="${1:-v5.29.5}"
VERSION="${1:-v5.31.0}"
TARGET_DIR="server/router/swagger-ui-dist"
TARBALL_URL="https://github.com/swagger-api/swagger-ui/archive/refs/tags/${VERSION}.tar.gz"
@@ -41,11 +41,19 @@ fi
echo ">> Patching swagger-initializer.js to point at /swagger.json"
sed -i -E \
if sed --version >/dev/null 2>&1; then
SED_INPLACE=(-i)
else
SED_INPLACE=(-i '')
fi
append_validator=$'/url:[[:space:]]*"[^"]*swagger\\.json"[[:space:]]*,?$/a\\\n validatorUrl: null,'
sed "${SED_INPLACE[@]}" -E \
-e 's#configUrl:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \
-e 's#url:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \
-e 's#urls:[[:space:]]*\[[^]]*\]#url: "/swagger.json"#' \
-e '/url:[[:space:]]*"[^\"]*swagger\.json"[[:space:]]*,?$/a\ validatorUrl: null,' \
-e "$append_validator" \
"$INDEX"
echo ">> Done. Files are in ${TARGET_DIR}"

View File

@@ -6,7 +6,6 @@ import (
"fmt"
"net/http"
"net/url"
"strings"
"vctp/components/views"
"vctp/internal/report"
@@ -22,7 +21,7 @@ import (
// @Failure 500 {string} string "Server error"
// @Router /snapshots/hourly [get]
func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) {
h.renderSnapshotList(w, r, "inventory_daily_", "Hourly Inventory Snapshots", views.SnapshotHourlyList)
h.renderSnapshotList(w, r, "hourly", "Hourly Inventory Snapshots", views.SnapshotHourlyList)
}
// SnapshotDailyList renders the daily snapshot list page.
@@ -34,7 +33,7 @@ func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) {
// @Failure 500 {string} string "Server error"
// @Router /snapshots/daily [get]
func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) {
h.renderSnapshotList(w, r, "inventory_daily_summary_", "Daily Inventory Snapshots", views.SnapshotDailyList)
h.renderSnapshotList(w, r, "daily", "Daily Inventory Snapshots", views.SnapshotDailyList)
}
// SnapshotMonthlyList renders the monthly snapshot list page.
@@ -46,7 +45,7 @@ func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) {
// @Failure 500 {string} string "Server error"
// @Router /snapshots/monthly [get]
func (h *Handler) SnapshotMonthlyList(w http.ResponseWriter, r *http.Request) {
h.renderSnapshotList(w, r, "inventory_monthly_summary_", "Monthly Inventory Snapshots", views.SnapshotMonthlyList)
h.renderSnapshotList(w, r, "monthly", "Monthly Inventory Snapshots", views.SnapshotMonthlyList)
}
// SnapshotReportDownload streams a snapshot table as XLSX.
@@ -91,28 +90,28 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request)
w.Write(reportData)
}
func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, prefix string, title string, renderer func([]views.SnapshotEntry) templ.Component) {
func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, snapshotType string, title string, renderer func([]views.SnapshotEntry) templ.Component) {
ctx := context.Background()
tables, err := report.ListTablesByPrefix(ctx, h.Database, prefix)
if err := report.EnsureSnapshotRegistry(ctx, h.Database); err != nil {
h.Logger.Error("Failed to ensure snapshot registry", "error", err)
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err)
return
}
records, err := report.ListSnapshots(ctx, h.Database, snapshotType)
if err != nil {
h.Logger.Error("Failed to list snapshot tables", "error", err, "prefix", prefix)
h.Logger.Error("Failed to list snapshots", "error", err, "snapshot_type", snapshotType)
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err)
return
}
entries := make([]views.SnapshotEntry, 0, len(tables))
for _, table := range tables {
if prefix == "inventory_daily_" && strings.HasPrefix(table, "inventory_daily_summary_") {
continue
}
label := table
if parsed, ok := report.FormatSnapshotLabel(prefix, table); ok {
label = parsed
}
entries := make([]views.SnapshotEntry, 0, len(records))
for _, record := range records {
label := report.FormatSnapshotLabel(snapshotType, record.SnapshotTime, record.TableName)
entries = append(entries, views.SnapshotEntry{
Label: label,
Link: "/api/report/snapshot?table=" + url.QueryEscape(table),
Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName),
})
}

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"vctp/db"
queries "vctp/db/queries"
models "vctp/server/models"
@@ -56,6 +57,17 @@ func (h *Handler) VmImport(w http.ResponseWriter, r *http.Request) {
//prettyPrint(inData)
}
if strings.HasPrefix(inData.Name, "vCLS-") {
h.Logger.Info("Skipping internal vCLS VM import", "vm_name", inData.Name)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]string{
"status": "OK",
"message": fmt.Sprintf("Skipped internal VM '%s'", inData.Name),
})
return
}
ctx := context.Background()
// Query Inventory table for this VM before adding it

View File

@@ -441,6 +441,14 @@ func (h *Handler) AddVmToInventory(evt models.CloudEventReceived, ctx context.Co
}
if strings.HasPrefix(vmObject.Name, "vCLS-") {
h.Logger.Info("Skipping internal vCLS VM", "vm_name", vmObject.Name)
if err := vc.Logout(); err != nil {
h.Logger.Error("unable to logout of vcenter", "error", err)
}
return 0, nil
}
//c.Logger.Debug("found VM")
srmPlaceholder = "FALSE" // Default assumption
//prettyPrint(vmObject)

View File

@@ -29,6 +29,9 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st
mux := http.NewServeMux()
mux.Handle("/assets/", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir))))
mux.Handle("/favicon.ico", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir))))
mux.Handle("/favicon-16x16.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir))))
mux.Handle("/favicon-32x32.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir))))
mux.HandleFunc("/", h.Home)
mux.HandleFunc("/api/event/vm/create", h.VmCreateEvent)
mux.HandleFunc("/api/event/vm/modify", h.VmModifyEvent)

View File

@@ -1,8 +0,0 @@
settings:
tenants_to_filter:
- "DecomVM"
node_charge_clusters:
- ".*CMD.*"
srm_activeactive_vms:
vcenter_addresses:
- "https://vc.lab.local/sdk"

View File

@@ -1 +1 @@
CPE_OPTS='-config /etc/dtms/vctp.yml -log-level info -log-output text'
CPE_OPTS='-settings /etc/dtms/vctp.yml'

View File

@@ -8,6 +8,7 @@ After=network.target
Type=simple
EnvironmentFile=/etc/default/vctp
User=vctp
Group=dtms
ExecStart=/usr/bin/vctp-linux-amd64 $CPE_OPTS
ExecStartPost=/usr/bin/sleep 3
Restart=always

View File

@@ -1,25 +1,24 @@
settings:
data_location: "/var/lib/cbs"
kickstart_location: "/var/lib/cbs/ks"
database_filename: "/var/lib/cbs/cbs.db"
log_level: "info"
log_output: "text"
database_driver: "sqlite"
database_url: "/var/lib/vctp/db.sqlite3"
bind_ip:
bind_port: 443
bind_port: 9443
bind_disable_tls: false
tls_cert_filename: "/etc/dtms/cbs.crt"
tls_key_filename: "/etc/dtms/cbs.key"
tftp_root_directory: "/var/lib/tftpboot"
tftp_images_subdirectory: "images"
replacements:
omapi:
key_name: "OMAPI"
key_secret:
special_files:
ldap_groups:
ldap_bind_address: ""
ldap_base_dn: ""
ldap_trust_cert_file: ""
ldap_disable_validation: false
ldap_insecure: false
auth_token_lifespan_hours: 2
auth_api_key: ""
tls_cert_filename: "/etc/dtms/vctp.crt"
tls_key_filename: "/etc/dtms/vctp.key"
vcenter_username: ""
vcenter_password: ""
vcenter_insecure: false
vcenter_event_polling_seconds: 60
vcenter_inventory_polling_seconds: 7200
vcenter_inventory_snapshot_seconds: 3600
vcenter_inventory_aggregate_seconds: 86400
hourly_snapshot_max_age_days: 60
daily_snapshot_max_age_months: 12
snapshot_cleanup_cron: "30 2 * * *"
tenants_to_filter:
node_charge_clusters:
srm_activeactive_vms:
vcenter_addresses: