From 7400e08c54eafa4c4ffccc68bdcd585050e0fc58 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Wed, 14 Jan 2026 09:28:30 +1100 Subject: [PATCH] updates --- .gitignore | 1 + README.md | 32 ++- components/core/header.templ | 55 ++--- components/core/header_templ.go | 6 +- components/views/index.templ | 6 +- components/views/index_templ.go | 8 +- components/views/snapshots.templ | 6 +- components/views/snapshots_templ.go | 14 +- .../20250115094500_snapshot_registry.sql | 14 ++ .../20250115094500_snapshot_registry.sql | 14 ++ db/schema.sql | 7 + dist/dist.go | 2 +- dist/favicon-16x16.png | Bin 0 -> 520 bytes dist/favicon-32x32.png | Bin 0 -> 1115 bytes dist/favicon.ico | Bin 0 -> 15406 bytes e2e/e2e_test.go.txt | 36 ++- go.mod | 19 +- go.sum | 47 +++- internal/report/snapshots.go | 155 +++++++++++-- internal/settings/settings.go | 74 +++++- internal/tasks/inventorySnapshots.go | 218 +++++++++++------- internal/tasks/monitorVcenter.go | 5 + internal/tasks/processEvents.go | 8 + internal/vcenter/vcenter.go | 15 +- log/log.go | 10 +- main.go | 209 ++++++++--------- scripts/drone.sh | 9 +- scripts/update-swagger-ui.sh | 16 +- server/handler/snapshots.go | 33 ++- server/handler/vmImport.go | 12 + server/handler/vmModifyEvent.go | 8 + server/router/router.go | 3 + src/vctp.default | 2 +- src/vctp.service | 1 + src/vctp.yml | 43 ++-- 35 files changed, 731 insertions(+), 357 deletions(-) create mode 100644 db/migrations/20250115094500_snapshot_registry.sql create mode 100644 db/migrations_postgres/20250115094500_snapshot_registry.sql create mode 100644 dist/favicon-16x16.png create mode 100644 dist/favicon-32x32.png create mode 100644 dist/favicon.ico diff --git a/.gitignore b/.gitignore index 4bf734e..8a44af5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.dylib vctp build/ +settings.yaml # Certificates *.pem diff --git a/README.md b/README.md index 8ce5c37..b8893df 100644 --- a/README.md +++ b/README.md @@ -28,27 +28,37 @@ Run `templ generate -path ./components` to generate code based on template files ## Documentation Run `swag init --exclude "pkg.mod,pkg.build,pkg.tools" -o server/router/docs` +#### Settings File +Configuration now lives in the YAML settings file. By default the service reads +`/etc/dtms/vctp.yml`, or you can override it with the `-settings` flag. + +```shell +vctp -settings /path/to/vctp.yml +``` + #### Database Configuration By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL -by setting environment variables: +by updating the settings file: -- `DB_DRIVER`: `sqlite` (default) or `postgres` -- `DB_URL`: SQLite file path/DSN or PostgreSQL DSN +- `settings.database_driver`: `sqlite` (default) or `postgres` +- `settings.database_url`: SQLite file path/DSN or PostgreSQL DSN Examples: -```shell -# SQLite (default) -DB_DRIVER=sqlite DB_URL=./db.sqlite3 +```yaml +settings: + database_driver: sqlite + database_url: ./db.sqlite3 -# PostgreSQL -DB_DRIVER=postgres DB_URL=postgres://user:pass@localhost:5432/vctp?sslmode=disable +settings: + database_driver: postgres + database_url: postgres://user:pass@localhost:5432/vctp?sslmode=disable ``` PostgreSQL migrations live in `db/migrations_postgres`, while SQLite migrations remain in `db/migrations`. #### Snapshot Retention -Hourly and daily snapshot table retention can be configured with environment variables: +Hourly and daily snapshot table retention can be configured in the settings file: -- `HOURLY_SNAPSHOT_MAX_AGE_DAYS` (default: 60) -- `DAILY_SNAPSHOT_MAX_AGE_MONTHS` (default: 12) \ No newline at end of file +- `settings.hourly_snapshot_max_age_days` (default: 60) +- `settings.daily_snapshot_max_age_months` (default: 12) diff --git a/components/core/header.templ b/components/core/header.templ index 4fa9e63..f7a9cd0 100644 --- a/components/core/header.templ +++ b/components/core/header.templ @@ -8,23 +8,25 @@ templ Header() { vCTP API + + + diff --git a/components/core/header_templ.go b/components/core/header_templ.go index 659f4b2..d261f00 100644 --- a/components/core/header_templ.go +++ b/components/core/header_templ.go @@ -31,20 +31,20 @@ func Header() templ.Component { templ_7745c5c3_Var1 = templ.NopComponent } ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "vCTP APIvCTP API") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\" rel=\"stylesheet\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/components/views/index.templ b/components/views/index.templ index 039aa71..3e6c4bb 100644 --- a/components/views/index.templ +++ b/components/views/index.templ @@ -20,10 +20,10 @@ templ Index(info BuildInfo) {
vCTP Console
-

Build Intelligence Dashboard

-

A glossy, snapshot-ready view of what is running.

+

Chargeback Intelligence Dashboard

+

Point in time snapshots of consumption.

-
+
Hourly Snapshots Daily Snapshots Monthly Snapshots diff --git a/components/views/index_templ.go b/components/views/index_templ.go index 604955d..34b411f 100644 --- a/components/views/index_templ.go +++ b/components/views/index_templ.go @@ -47,14 +47,14 @@ func Index(info BuildInfo) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
vCTP Console

Build Intelligence Dashboard

A glossy, snapshot-ready view of what is running.

Hourly Snapshots Daily Snapshots Monthly Snapshots Swagger UI

Build Time

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

vCTP Console

Chargeback Intelligence Dashboard

Point in time snapshots of consumption.

Hourly Snapshots Daily Snapshots Monthly Snapshots Swagger UI

Build Time

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(info.BuildTime) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 38, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 38, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) if templ_7745c5c3_Err != nil { @@ -67,7 +67,7 @@ func Index(info BuildInfo) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(info.SHA1Ver) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 42, Col: 57} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 42, Col: 57} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { @@ -80,7 +80,7 @@ func Index(info BuildInfo) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(info.GoVersion) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 46, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 46, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) if templ_7745c5c3_Err != nil { diff --git a/components/views/snapshots.templ b/components/views/snapshots.templ index c67d3a6..15d7d58 100644 --- a/components/views/snapshots.templ +++ b/components/views/snapshots.templ @@ -32,15 +32,15 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {

Snapshot Library

{title}

-

{subtitle}

+

{subtitle}

Back to Dashboard
-
-

Available Exports

+
+

Available Exports 

{len(entries)} files
    diff --git a/components/views/snapshots_templ.go b/components/views/snapshots_templ.go index f4ea93c..549d398 100644 --- a/components/views/snapshots_templ.go +++ b/components/views/snapshots_templ.go @@ -140,33 +140,33 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 34, Col: 49} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 35, Col: 51} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "

Back to Dashboard

Available Exports

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "

Back to Dashboard

Available Exports 

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 44, Col: 83} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { @@ -184,7 +184,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 49, Col: 71} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { @@ -197,7 +197,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var9 templ.SafeURL templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 50, Col: 45} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { diff --git a/db/migrations/20250115094500_snapshot_registry.sql b/db/migrations/20250115094500_snapshot_registry.sql new file mode 100644 index 0000000..f54d821 --- /dev/null +++ b/db/migrations/20250115094500_snapshot_registry.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE snapshot_registry; +-- +goose StatementEnd diff --git a/db/migrations_postgres/20250115094500_snapshot_registry.sql b/db/migrations_postgres/20250115094500_snapshot_registry.sql new file mode 100644 index 0000000..69fa174 --- /dev/null +++ b/db/migrations_postgres/20250115094500_snapshot_registry.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id BIGSERIAL PRIMARY KEY, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE snapshot_registry; +-- +goose StatementEnd diff --git a/db/schema.sql b/db/schema.sql index 33f91ca..a646637 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -66,3 +66,10 @@ CREATE TABLE IF NOT EXISTS inventory_history ( "PreviousResourcePool" TEXT, "PreviousProvisionedDisk" REAL ); + +CREATE TABLE IF NOT EXISTS snapshot_registry ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "snapshot_type" TEXT NOT NULL, + "table_name" TEXT NOT NULL UNIQUE, + "snapshot_time" INTEGER NOT NULL +); diff --git a/dist/dist.go b/dist/dist.go index 02c066e..01e24de 100644 --- a/dist/dist.go +++ b/dist/dist.go @@ -4,5 +4,5 @@ import ( "embed" ) -//go:embed all:assets +//go:embed all:assets favicon.ico favicon-16x16.png favicon-32x32.png var AssetsDir embed.FS diff --git a/dist/favicon-16x16.png b/dist/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..e0f8e680579eb67d01bf59e499175faf524481c4 GIT binary patch literal 520 zcmV+j0{8uiP) zy-Px26vm$tD=Z?h$Y^LOB3i^+8rmwMC1~znXmu$PqSo9L{{le}cir5??~6JFwU~n# zYKrDZ_u6|-3HS9Q2z*}O^PF>j@45HAvq0cZs4F!1r>N&n0B{3fP6!6W3=R&=I6GU1 z)w)2IB(vFw%garcsHL780?ZkWvv_#eM<#QMMB)U!o`y~0000< KMNUMnLSTXipx~AO literal 0 HcmV?d00001 diff --git a/dist/favicon-32x32.png b/dist/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..0931913005e447422de9dc392a6eed21dd4ffa2a GIT binary patch literal 1115 zcmV-h1f=_kP)cc)F!;heZ9zQ`pq9GJPK_5v)MMXrB z^!OV35PX54B$}j-rGw8T%~x7JN5|>xV{_(S=iWP431Rj*`|Pv!-fPZ1j~xNzGkKQ_ zyzOD)Um{_-C}(~IOTHrq@ZSJnWwKO1mWy)k$YBKhmzFAPXdeG1MnL_aFJHb4SFdit znKSEo|LRpLva*swnu?ryw(or<`)C}#8E{rhf#pRbrqk+^bYGj85=uw?sDTzmi{EgI-*}-pC1bbc~1i$OsG#d6AY@itOxKJbv6J11Be&Wn-m5d;1t8UxM7+M)=uz zB1~an7iMQkQx+KkDWs}uP&$Oeu~IhH8pOv(^JOQ@%a@(hsfsFr*49ysj7$s1l$6y% z52x(i8^7V)Rx}BacX4rWAEc$lv)hfV%5wSgWuJIgpu0eN90Qlk{NUNy8xa!| zi92_`VR~BQ?w#itmH^?4ih8NzvUTeU-b)+y=3w8xHK?uqg(pwsZ-^A9r0XJ3S2rwt zv$B#A6(x5&eAt4cM-y=Vd<|x1{=GuNLtBwYQo&YM4u~16)g*hEXxZ=IcjCp14oHa# zhcJpR0(7K$dL~6Er*rPHA;X0W8$oqIe!iQA3l?1j=wYg;2=45Rj6|NuZZ{z>Z#$kp zZ%23c1h*};(M5oQmX`+!96V?N)r+@p??Oq*PgXB}A;_XcH-Xa9K9PXF`=duroI14* z*RIuz>PG+;py(z*1qvOrK2(vJRB+#YPUu1Ujw*RTKeSIWQpZv8z_`@2d@skgq~P&ez+OKC?|mf@Qfjl{za zqtW4EYFWC}r*3O*E-gFvT%YeM9gH`iX2AlVdYj*WpK5>l$>+O@zy7)B1pbDXUh=7z zb=hS!v}X^+O_<WL?+H1st9*+4({#v3W+xZ~71;u&*c&7F6O%i9jWYf!c+z+K>4 zDg_rGp=Vu8wQTl&_Soh2`I{)FmK1w|FP^$m!H`=ynk>fp3 z9ZR>*;3J_qJzcap<%APxpuL?cScki%=c+#M@t)>p$)2zdB>sjMUZBC1x=81haL_^2y>+YDzrw}OOs9Cx9GyM%5B62HPRU{nODzxex$?^| zb^exiACFH9&k3wgtkY2WneoxT5{e&v)RI1QGGN`a&svt7X*zw+IPW~kTHG_vKi`sm z@{}orbqO2MJU&i&S=EDmC!wddHsD&XUARzGg`H$YE0}Na6(ltL@kc7V@YW3lTW&$1D_61_02bq@sf@_lG+OjsOJ0cNgaF8aiY^O_dAM; z2s*RRYcIe)i5)-Qkqpo`dAHq0*kh`=zcJ?|l)v{Lp)cXHs{N!sCvZ#!a~}MgQd4P^ z#~*)w7mtUsB`|lTb#7>GHddM-2oa?VA>>1b-tH1r0npvhq zjzLNK-n%Ow$b@UA<@R0pp<$-!SO$|1JeK{S```ZJ0{@loh?8)LR-40Dpg0K^YdlXT zEz-2vJZ|}X_w1{AW5?GkTLEN+%u;ridp^JS+u=u1xyLKB=y#SYYWqAupY?mkp&&pV z$2wzQ&DEN-NtaORoEcIy}iOND}VRhs2($`KE!#0g#NZR%UHG* zUbK8`PD_*Gi+f{S9KmLb7Wo3nfw{kX(ULybeYJ=^8ydhby$ubN zIemIm%Z_*mY~azMA%pnv({XmRBQw*nZx?8cBUP5f=Sxk=$p+c+QvKa`;zGMtvCkJN z{-M%c!hQ$*02M|C2MPXC_ElF2U#X4#+vH=8iOM$M+|WFC+Vb**tcW)tK8dqs_>DLV z47I<)jgP&`3>#j2QI~V^q)F7t=P6hVTR-_k{~m5Rf~D7$I(bw2W$V~`$Uj{?3dj5Qa{8-1xL$T~%ZUf>)$Z@Iq`fEZwB^aLJ z!inR!(ax1C4f1<-><~KBACI(CPo>@E7j1-tVzwU2Q213e1Bc{lbEwLH?#i2ItM;Qu*u4F)N8VAuu8!(?#`jLEw-tVcaGj>1AkrRYImuG0OL2y1ANx3C$oE>2gRRMNE zFlVAz2U3S^eC3sZ=g*RVsQ03WA67*d&p68qCdVV?_~XTy5QubFR1k8wLY2=Be>0yI znV~f&Cn*0)=J|jg^l9m9uSHasB3HGexY#T{HKn8w{9wDbo6B)#i?cKdJF~M1xf14j z;d)W^wjqF9+L0ICGPE2tyoyjHGRiGk%Zv{9N`KbRhBxg3YbLy4h1z zrIV{>KRI0eZKhNBzyo?;Nrs5o`perf<6FM-7$DXt0lLi6e+bp4V?L#CyrJtm$q(b< zB=^EncCSo`U8?}wJW%~$%8-jz^TQ7+8}SSuz?56=YysdAa^zp9e-u`&q?3*?YbXhpv^iNITazrC|KX%{|v|>#@tzc`Q<{-nPZ)u z>( 0 + if !canDetectMissing { + c.Logger.Warn("no VMs returned from vcenter; skipping missing VM detection", "url", url) + } inventoryRows, err := c.Database.Queries().GetInventoryByVcenter(ctx, url) if err != nil { @@ -116,8 +127,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo presentSnapshots[vm.Reference().Value] = row totals.VmCount++ - totals.VcpuTotal += nullInt64ToInt(row.InitialVcpus) - totals.RamTotal += nullInt64ToInt(row.InitialRam) + totals.VcpuTotal += nullInt64ToInt(row.VcpuCount) + totals.RamTotal += nullInt64ToInt(row.RamGB) totals.DiskTotal += nullFloat64ToFloat(row.ProvisionedDisk) } @@ -127,7 +138,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo } } + if !canDetectMissing { + vc.Logout() + continue + } + for _, inv := range inventoryRows { + if strings.HasPrefix(inv.Name, "vCLS-") { + continue + } vmID := inv.VmId.String if vmID != "" { if _, ok := presentSnapshots[vmID]; ok { @@ -137,6 +156,17 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo row := snapshotFromInventory(inv, startTime) row.IsPresent = "FALSE" + if !row.DeletionTime.Valid { + deletionTime := startTime.Unix() + row.DeletionTime = sql.NullInt64{Int64: deletionTime, Valid: true} + if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{ + DeletionTime: row.DeletionTime, + VmId: inv.VmId, + DatacenterName: inv.Datacenter, + }); err != nil { + c.Logger.Warn("failed to mark inventory record deleted", "error", err, "vm_id", row.VmId.String) + } + } if err := insertDailyInventoryRow(ctx, dbConn, tableName, row); err != nil { c.Logger.Error("failed to insert missing VM snapshot", "error", err, "vm_id", row.VmId.String) } @@ -148,7 +178,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo "vcenter", url, "vm_count", totals.VmCount, "vcpu_total", totals.VcpuTotal, - "ram_total_mb", totals.RamTotal, + "ram_total_gb", totals.RamTotal, "disk_total_gb", totals.DiskTotal, ) } @@ -160,7 +190,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo // RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table. func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error { targetTime := time.Now().Add(-time.Minute) - sourceTable, err := dailyInventoryTableName(targetTime) + sourceTable, err := hourlyInventoryTableName(targetTime) if err != nil { return err } @@ -173,6 +203,9 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo if err := ensureDailySummaryTable(ctx, dbConn, summaryTable); err != nil { return err } + if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { + return err + } currentTotals, err := snapshotTotalsForTable(ctx, dbConn, sourceTable) if err != nil { @@ -182,12 +215,12 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo "table", sourceTable, "vm_count", currentTotals.VmCount, "vcpu_total", currentTotals.VcpuTotal, - "ram_total_mb", currentTotals.RamTotal, + "ram_total_gb", currentTotals.RamTotal, "disk_total_gb", currentTotals.DiskTotal, ) } - prevTable, _ := dailyInventoryTableName(targetTime.AddDate(0, 0, -1)) + prevTable, _ := hourlyInventoryTableName(targetTime.AddDate(0, 0, -1)) if prevTable != "" && tableExists(ctx, dbConn, prevTable) { prevTotals, err := snapshotTotalsForTable(ctx, dbConn, prevTable) if err != nil { @@ -198,7 +231,7 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo "previous_table", prevTable, "vm_delta", currentTotals.VmCount-prevTotals.VmCount, "vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal, - "ram_delta_mb", currentTotals.RamTotal-prevTotals.RamTotal, + "ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal, "disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal, ) } @@ -207,19 +240,19 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "SamplesPresent", "AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct" ) SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", @@ -232,14 +265,17 @@ SELECT FROM %s GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, summaryTable, sourceTable) if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate daily inventory", "error", err, "source_table", sourceTable) return err } + if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, targetTime); err != nil { + c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable) + } c.Logger.Debug("Finished daily inventory aggregation", "source_table", sourceTable, "summary_table", summaryTable) return nil @@ -251,7 +287,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) targetMonth := firstOfThisMonth.AddDate(0, -1, 0) - monthPrefix := fmt.Sprintf("inventory_daily_%s", targetMonth.Format("200601")) + monthPrefix := fmt.Sprintf("inventory_hourly_%s", targetMonth.Format("200601")) dailyTables, err := report.ListTablesByPrefix(ctx, c.Database, monthPrefix) if err != nil { return err @@ -269,11 +305,14 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. if err := ensureMonthlySummaryTable(ctx, dbConn, monthlyTable); err != nil { return err } + if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { + return err + } unionQuery := buildUnionQuery(dailyTables, []string{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, `"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, - `"ProvisionedDisk"`, `"InitialVcpus"`, `"InitialRam"`, `"IsTemplate"`, `"PoweredOn"`, + `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, `"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`, }) if strings.TrimSpace(unionQuery) == "" { @@ -288,7 +327,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. "month", targetMonth.Format("2006-01"), "vm_count", monthlyTotals.VmCount, "vcpu_total", monthlyTotals.VcpuTotal, - "ram_total_mb", monthlyTotals.RamTotal, + "ram_total_gb", monthlyTotals.RamTotal, "disk_total_gb", monthlyTotals.DiskTotal, ) } @@ -296,18 +335,18 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct" ) SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - AVG(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus", - AVG(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam", - AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", @@ -322,14 +361,17 @@ FROM ( ) snapshots GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, monthlyTable, unionQuery) if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01")) return err } + if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil { + c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable) + } c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable) return nil @@ -338,15 +380,15 @@ GROUP BY // RunSnapshotCleanup drops hourly and daily snapshot tables older than retention. func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error { now := time.Now() - hourlyMaxDays := getEnvInt("HOURLY_SNAPSHOT_MAX_AGE_DAYS", 60) - dailyMaxMonths := getEnvInt("DAILY_SNAPSHOT_MAX_AGE_MONTHS", 12) + hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60) + dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12) hourlyCutoff := now.AddDate(0, 0, -hourlyMaxDays) dailyCutoff := now.AddDate(0, -dailyMaxMonths, 0) dbConn := c.Database.DB() - hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_daily_") + hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_hourly_") if err != nil { return err } @@ -356,7 +398,7 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) if strings.HasPrefix(table, "inventory_daily_summary_") { continue } - tableDate, ok := parseSnapshotDate(table, "inventory_daily_", "20060102") + tableDate, ok := parseSnapshotDate(table, "inventory_hourly_", "2006010215") if !ok { continue } @@ -365,6 +407,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) c.Logger.Error("failed to drop hourly snapshot table", "error", err, "table", table) } else { removedHourly++ + if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil { + c.Logger.Warn("failed to remove hourly snapshot registry entry", "error", err, "table", table) + } } } } @@ -384,6 +429,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) c.Logger.Error("failed to drop daily snapshot table", "error", err, "table", table) } else { removedDaily++ + if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil { + c.Logger.Warn("failed to remove daily snapshot registry entry", "error", err, "table", table) + } } } } @@ -397,8 +445,8 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) return nil } -func dailyInventoryTableName(t time.Time) (string, error) { - return safeTableName(fmt.Sprintf("inventory_daily_%s", t.Format("20060102"))) +func hourlyInventoryTableName(t time.Time) (string, error) { + return safeTableName(fmt.Sprintf("inventory_hourly_%s", t.Format("2006010215"))) } func dailySummaryTableName(t time.Time) (string, error) { @@ -435,8 +483,8 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, @@ -445,8 +493,14 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "IsPresent" TEXT NOT NULL );`, tableName) - _, err := dbConn.ExecContext(ctx, ddl) - return err + if _, err := dbConn.ExecContext(ctx, ddl); err != nil { + return err + } + + return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ + {Name: "VcpuCount", Type: "BIGINT"}, + {Name: "RamGB", Type: "BIGINT"}, + }) } func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { @@ -465,16 +519,16 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, "VmUuid" TEXT, "SamplesPresent" BIGINT NOT NULL, - "AvgVcpus" REAL, - "AvgRam" REAL, - "AvgDisk" REAL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, "AvgIsPresent" REAL, "PoolTinPct" REAL, "PoolBronzePct" REAL, @@ -487,9 +541,9 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str } return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgVcpus", Type: "REAL"}, - {Name: "AvgRam", Type: "REAL"}, - {Name: "AvgDisk", Type: "REAL"}, + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, {Name: "AvgIsPresent", Type: "REAL"}, {Name: "PoolTinPct", Type: "REAL"}, {Name: "PoolBronzePct", Type: "REAL"}, @@ -514,15 +568,15 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, "VmUuid" TEXT, - "AvgVcpus" REAL, - "AvgRam" REAL, - "AvgDisk" REAL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, "AvgIsPresent" REAL, "PoolTinPct" REAL, "PoolBronzePct" REAL, @@ -535,7 +589,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s } return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgDisk", Type: "REAL"}, + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, + {Name: "AvgIsPresent", Type: "REAL"}, {Name: "PoolTinPct", Type: "REAL"}, {Name: "PoolBronzePct", Type: "REAL"}, {Name: "PoolSilverPct", Type: "REAL"}, @@ -622,8 +679,8 @@ func snapshotTotalsForTable(ctx context.Context, dbConn *sqlx.DB, table string) query := fmt.Sprintf(` SELECT COUNT(DISTINCT "VmId") AS vm_count, - COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total, - COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total, + COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total, + COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total, COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total FROM %s WHERE "IsPresent" = 'TRUE' @@ -640,8 +697,8 @@ func snapshotTotalsForUnion(ctx context.Context, dbConn *sqlx.DB, unionQuery str query := fmt.Sprintf(` SELECT COUNT(DISTINCT "VmId") AS vm_count, - COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total, - COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total, + COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total, + COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total, COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total FROM ( %s @@ -694,13 +751,8 @@ func nullFloat64ToFloat(value sql.NullFloat64) float64 { return 0 } -func getEnvInt(key string, fallback int) int { - raw := strings.TrimSpace(os.Getenv(key)) - if raw == "" { - return fallback - } - value, err := strconv.Atoi(raw) - if err != nil || value < 0 { +func intWithDefault(value int, fallback int) int { + if value <= 0 { return fallback } return value @@ -731,8 +783,8 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi if !vmObject.Config.CreateDate.IsZero() { row.CreationTime = sql.NullInt64{Int64: vmObject.Config.CreateDate.Unix(), Valid: true} } - row.InitialVcpus = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0} - row.InitialRam = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB), Valid: vmObject.Config.Hardware.MemoryMB > 0} + row.VcpuCount = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0} + row.RamGB = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB) / 1024, Valid: vmObject.Config.Hardware.MemoryMB > 0} totalDiskBytes := int64(0) for _, device := range vmObject.Config.Hardware.Device { @@ -774,11 +826,11 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi if !row.ProvisionedDisk.Valid { row.ProvisionedDisk = inv.ProvisionedDisk } - if !row.InitialVcpus.Valid { - row.InitialVcpus = inv.InitialVcpus + if !row.VcpuCount.Valid { + row.VcpuCount = inv.InitialVcpus } - if !row.InitialRam.Valid { - row.InitialRam = inv.InitialRam + if !row.RamGB.Valid && inv.InitialRam.Valid { + row.RamGB = sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Int64 > 0} } if row.IsTemplate == "" { row.IsTemplate = boolStringFromInterface(inv.IsTemplate) @@ -837,8 +889,8 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent Cluster: inv.Cluster, Folder: inv.Folder, ProvisionedDisk: inv.ProvisionedDisk, - InitialVcpus: inv.InitialVcpus, - InitialRam: inv.InitialRam, + VcpuCount: inv.InitialVcpus, + RamGB: sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Valid && inv.InitialRam.Int64 > 0}, IsTemplate: boolStringFromInterface(inv.IsTemplate), PoweredOn: boolStringFromInterface(inv.PoweredOn), SrmPlaceholder: boolStringFromInterface(inv.SrmPlaceholder), @@ -851,8 +903,8 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str query := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent" + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent" ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); `, tableName) @@ -874,8 +926,8 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); row.Cluster, row.Folder, row.ProvisionedDisk, - row.InitialVcpus, - row.InitialRam, + row.VcpuCount, + row.RamGB, row.IsTemplate, row.PoweredOn, row.SrmPlaceholder, diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index 0af5f6e..1710458 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -262,6 +262,11 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce return errors.New("can't process empty vm object") } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + c.Logger.Debug("Skipping internal vCLS VM", "vm_name", vmObject.Name) + return nil + } + c.Logger.Debug("found VM") /* diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index dd45035..377e6b8 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -83,6 +83,14 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { continue } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + c.Logger.Info("Skipping internal vCLS VM event", "vm_name", vmObject.Name) + if err := c.Database.Queries().UpdateEventsProcessed(ctx, evt.Eid); err != nil { + c.Logger.Error("Unable to mark vCLS event as processed", "event_id", evt.Eid, "error", err) + } + continue + } + //c.Logger.Debug("found VM") srmPlaceholder = "FALSE" // Default assumption //prettyPrint(vmObject) diff --git a/internal/vcenter/vcenter.go b/internal/vcenter/vcenter.go index e5a7796..8a641fb 100644 --- a/internal/vcenter/vcenter.go +++ b/internal/vcenter/vcenter.go @@ -6,7 +6,6 @@ import ( "log" "log/slog" "net/url" - "os" "path" "strings" @@ -30,6 +29,7 @@ type Vcenter struct { type VcenterLogin struct { Username string Password string + Insecure bool } type VmProperties struct { @@ -51,13 +51,6 @@ func New(logger *slog.Logger, creds *VcenterLogin) *Vcenter { } func (v *Vcenter) Login(vUrl string) error { - var insecure bool - - // TODO - fix this - insecureString := os.Getenv("VCENTER_INSECURE") - //username := os.Getenv("VCENTER_USERNAME") - //password := os.Getenv("VCENTER_PASSWORD") - // Connect to vCenter u, err := soap.ParseURL(vUrl) if err != nil { @@ -74,11 +67,7 @@ func (v *Vcenter) Login(vUrl string) error { } */ - if insecureString == "true" { - insecure = true - } - - c, err := govmomi.NewClient(v.ctx, u, insecure) + c, err := govmomi.NewClient(v.ctx, u, v.credentials.Insecure) if err != nil { v.Logger.Error("Unable to connect to vCenter", "error", err) return fmt.Errorf("unable to connect to vCenter : %s", err) diff --git a/log/log.go b/log/log.go index 2a17a2f..05a000f 100644 --- a/log/log.go +++ b/log/log.go @@ -65,10 +65,9 @@ func ToLevel(level string) Level { } } -// GetLevel returns the log level from the environment variable. +// GetLevel returns the default log level. func GetLevel() Level { - level := os.Getenv("LOG_LEVEL") - return ToLevel(level) + return LevelInfo } // Output represents the log output. @@ -93,8 +92,7 @@ func ToOutput(output string) Output { } } -// GetOutput returns the log output from the environment variable. +// GetOutput returns the default log output. func GetOutput() Output { - output := os.Getenv("LOG_OUTPUT") - return ToOutput(output) + return OutputText } diff --git a/main.go b/main.go index 6d82f01..a6ff273 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,8 @@ package main import ( "context" + "flag" "fmt" - "log/slog" "os" "runtime" "strings" @@ -19,7 +19,6 @@ import ( "vctp/server/router" "github.com/go-co-op/gocron/v2" - "github.com/joho/godotenv" ) var ( @@ -34,22 +33,29 @@ var ( ) func main() { - // Load data from environment file - envFilename := utils.GetFilePath(".env") - err := godotenv.Load(envFilename) - if err != nil { - panic("Error loading .env file") - } + settingsPath := flag.String("settings", "/etc/dtms/vctp.yml", "Path to settings YAML") + flag.Parse() - logger := log.New( - log.GetLevel(), - log.GetOutput(), - ) + bootstrapLogger := log.New(log.LevelInfo, log.OutputText) ctx, cancel := context.WithCancel(context.Background()) + // Load settings from yaml + s := settings.New(bootstrapLogger, *settingsPath) + err := s.ReadYMLSettings() + if err != nil { + bootstrapLogger.Error("failed to open yaml settings file", "error", err, "filename", *settingsPath) + os.Exit(1) + } + + logger := log.New( + log.ToLevel(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogLevel))), + log.ToOutput(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogOutput))), + ) + s.Logger = logger + // Configure database - dbDriver := os.Getenv("DB_DRIVER") + dbDriver := strings.TrimSpace(s.Values.Settings.DatabaseDriver) if dbDriver == "" { dbDriver = "sqlite" } @@ -57,12 +63,12 @@ func main() { if normalizedDriver == "" || normalizedDriver == "sqlite3" { normalizedDriver = "sqlite" } - dbURL := os.Getenv("DB_URL") + dbURL := strings.TrimSpace(s.Values.Settings.DatabaseURL) if dbURL == "" && normalizedDriver == "sqlite" { dbURL = utils.GetFilePath("db.sqlite3") } - database, err := db.New(logger, db.Config{Driver: dbDriver, DSN: dbURL}) + database, err := db.New(logger, db.Config{Driver: normalizedDriver, DSN: dbURL}) if err != nil { logger.Error("Failed to create database", "error", err) os.Exit(1) @@ -70,56 +76,36 @@ func main() { defer database.Close() //defer database.DB().Close() - if err = db.Migrate(database, dbDriver); err != nil { + if err = db.Migrate(database, normalizedDriver); err != nil { logger.Error("failed to migrate database", "error", err) os.Exit(1) } - // Load settings from yaml - settingsFile := os.Getenv("SETTINGS_FILE") - if settingsFile == "" { - settingsFile = "settings.yaml" - } - - // TODO - how to pass this to the other packages that will need this info? - s := settings.New(logger, settingsFile) - err = s.ReadYMLSettings() - //s, err := settings.ReadYMLSettings(logger, settingsFile) - if err != nil { - logger.Error("failed to open yaml settings file", "error", err, "filename", settingsFile) - //os.Exit(1) - } else { - logger.Debug("Loaded yaml settings", "contents", s) - } - // Determine bind IP - bindIP := os.Getenv("BIND_IP") + bindIP := strings.TrimSpace(s.Values.Settings.BindIP) if bindIP == "" { bindIP = utils.GetOutboundIP().String() } // Determine bind port - bindPort := os.Getenv("BIND_PORT") - if bindPort == "" { - bindPort = "9443" + bindPort := s.Values.Settings.BindPort + if bindPort == 0 { + bindPort = 9443 } bindAddress := fmt.Sprint(bindIP, ":", bindPort) //logger.Info("Will listen on address", "ip", bindIP, "port", bindPort) // Determine bind disable TLS - bindDisableTlsEnv := os.Getenv("BIND_DISABLE_TLS") - if bindDisableTlsEnv == "true" { - bindDisableTls = true - } + bindDisableTls = s.Values.Settings.BindDisableTLS // Get file names for TLS cert/key - tlsCertFilename := os.Getenv("TLS_CERT_FILE") + tlsCertFilename := strings.TrimSpace(s.Values.Settings.TLSCertFilename) if tlsCertFilename != "" { tlsCertFilename = utils.GetFilePath(tlsCertFilename) } else { tlsCertFilename = "./cert.pem" } - tlsKeyFilename := os.Getenv("TLS_KEY_FILE") + tlsKeyFilename := strings.TrimSpace(s.Values.Settings.TLSKeyFilename) if tlsKeyFilename != "" { tlsKeyFilename = utils.GetFilePath(tlsKeyFilename) } else { @@ -132,9 +118,9 @@ func main() { utils.GenerateCerts(tlsCertFilename, tlsKeyFilename) } - // Load vcenter credentials from .env + // Load vcenter credentials from serttings, decrypt if required a := secrets.New(logger, encryptionKey) - vcEp := os.Getenv("VCENTER_PASSWORD") + vcEp := strings.TrimSpace(s.Values.Settings.VcenterPassword) if len(vcEp) == 0 { logger.Error("No vcenter password configured") os.Exit(1) @@ -143,13 +129,27 @@ func main() { if err != nil { logger.Error("failed to decrypt vcenter credentials. Assuming un-encrypted", "error", err) vcPass = []byte(vcEp) + if cipherText, encErr := a.Encrypt([]byte(vcEp)); encErr != nil { + logger.Warn("failed to encrypt vcenter credentials", "error", encErr) + } else { + s.Values.Settings.VcenterPassword = cipherText + if err := s.WriteYMLSettings(); err != nil { + logger.Warn("failed to update settings with encrypted vcenter password", "error", err) + } else { + logger.Info("encrypted vcenter password stored in settings file") + } + } //os.Exit(1) } creds := vcenter.VcenterLogin{ - //insecureString := os.Getenv("VCENTER_INSECURE") - Username: os.Getenv("VCENTER_USERNAME"), + Username: strings.TrimSpace(s.Values.Settings.VcenterUsername), Password: string(vcPass), + Insecure: s.Values.Settings.VcenterInsecure, + } + if creds.Username == "" { + logger.Error("No vcenter username configured") + os.Exit(1) } // Prepare the task scheduler @@ -167,83 +167,51 @@ func main() { VcCreds: &creds, } - cronFrequencyString := os.Getenv("VCENTER_EVENT_POLLING_SECONDS") - if cronFrequencyString != "" { - cronFrequency, err = time.ParseDuration(cronFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_EVENT_POLLING_SECONDS value to time duration. Defaulting to 60s", "value", cronFrequencyString, "error", err) - cronFrequency = time.Second * 60 - } - } else { - cronFrequency = time.Second * 60 - } + cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60) logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency) - cronInventoryFrequencyString := os.Getenv("VCENTER_INVENTORY_POLLING_SECONDS") - if cronInventoryFrequencyString != "" { - cronInvFrequency, err = time.ParseDuration(cronInventoryFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 7200", "value", cronInventoryFrequencyString, "error", err) - cronInvFrequency = time.Second * 7200 - } - } else { - cronInvFrequency = time.Second * 7200 - } + cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200) logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency) - cronSnapshotFrequencyString := os.Getenv("VCENTER_INVENTORY_SNAPSHOT_SECONDS") - if cronSnapshotFrequencyString != "" { - cronSnapshotFrequency, err = time.ParseDuration(cronSnapshotFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_SNAPSHOT_SECONDS value to time duration. Defaulting to 3600", "value", cronSnapshotFrequencyString, "error", err) - cronSnapshotFrequency = time.Hour - } - } else { - cronSnapshotFrequency = time.Hour - } + cronSnapshotFrequency = durationFromSeconds(s.Values.Settings.VcenterInventorySnapshotSeconds, 3600) logger.Debug("Setting VM inventory snapshot cronjob frequency to", "frequency", cronSnapshotFrequency) - cronAggregateFrequencyString := os.Getenv("VCENTER_INVENTORY_AGGREGATE_SECONDS") - if cronAggregateFrequencyString != "" { - cronAggregateFrequency, err = time.ParseDuration(cronAggregateFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_AGGREGATE_SECONDS value to time duration. Defaulting to 86400", "value", cronAggregateFrequencyString, "error", err) - cronAggregateFrequency = time.Hour * 24 - } - } else { - cronAggregateFrequency = time.Hour * 24 - } + cronAggregateFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryAggregateSeconds, 86400) logger.Debug("Setting VM inventory aggregation cronjob frequency to", "frequency", cronAggregateFrequency) - // start background processing for events stored in events table - startsAt := time.Now().Add(time.Second * 10) - job, err := c.NewJob( - gocron.DurationJob(cronFrequency), - gocron.NewTask(func() { - ct.RunVmCheck(ctx, logger) - }), gocron.WithSingletonMode(gocron.LimitModeReschedule), - gocron.WithStartAt(gocron.WithStartDateTime(startsAt)), - ) - if err != nil { - logger.Error("failed to start event processing cron job", "error", err) - os.Exit(1) - } - logger.Debug("Created event processing cron job", "job", job.ID(), "starting_at", startsAt) + /* + // start background processing for events stored in events table + startsAt := time.Now().Add(time.Second * 10) + job, err := c.NewJob( + gocron.DurationJob(cronFrequency), + gocron.NewTask(func() { + ct.RunVmCheck(ctx, logger) + }), gocron.WithSingletonMode(gocron.LimitModeReschedule), + gocron.WithStartAt(gocron.WithStartDateTime(startsAt)), + ) + if err != nil { + logger.Error("failed to start event processing cron job", "error", err) + os.Exit(1) + } + logger.Debug("Created event processing cron job", "job", job.ID(), "starting_at", startsAt) + */ // start background checks of vcenter inventory - startsAt2 := time.Now().Add(cronInvFrequency) - job2, err := c.NewJob( - gocron.DurationJob(cronInvFrequency), - gocron.NewTask(func() { - ct.RunVcenterPoll(ctx, logger) - }), gocron.WithSingletonMode(gocron.LimitModeReschedule), - gocron.WithStartAt(gocron.WithStartDateTime(startsAt2)), - ) - if err != nil { - logger.Error("failed to start vcenter inventory cron job", "error", err) - os.Exit(1) - } - logger.Debug("Created vcenter inventory cron job", "job", job2.ID(), "starting_at", startsAt2) + /* + startsAt2 := time.Now().Add(cronInvFrequency) + job2, err := c.NewJob( + gocron.DurationJob(cronInvFrequency), + gocron.NewTask(func() { + ct.RunVcenterPoll(ctx, logger) + }), gocron.WithSingletonMode(gocron.LimitModeReschedule), + gocron.WithStartAt(gocron.WithStartDateTime(startsAt2)), + ) + if err != nil { + logger.Error("failed to start vcenter inventory cron job", "error", err) + os.Exit(1) + } + logger.Debug("Created vcenter inventory cron job", "job", job2.ID(), "starting_at", startsAt2) + */ startsAt3 := time.Now().Add(cronSnapshotFrequency) if cronSnapshotFrequency == time.Hour { @@ -292,8 +260,12 @@ func main() { } logger.Debug("Created vcenter monthly aggregation cron job", "job", job5.ID()) + snapshotCleanupCron := strings.TrimSpace(s.Values.Settings.SnapshotCleanupCron) + if snapshotCleanupCron == "" { + snapshotCleanupCron = "30 2 * * *" + } job6, err := c.NewJob( - gocron.CronJob("0 30 2 * *", false), + gocron.CronJob(snapshotCleanupCron, false), gocron.NewTask(func() { ct.RunSnapshotCleanup(ctx, logger) }), gocron.WithSingletonMode(gocron.LimitModeReschedule), @@ -325,3 +297,10 @@ func main() { os.Exit(0) } + +func durationFromSeconds(value int, fallback int) time.Duration { + if value <= 0 { + return time.Second * time.Duration(fallback) + } + return time.Second * time.Duration(value) +} diff --git a/scripts/drone.sh b/scripts/drone.sh index 78cf3f4..5837726 100755 --- a/scripts/drone.sh +++ b/scripts/drone.sh @@ -10,8 +10,13 @@ buildtime=$(date +%Y-%m-%dT%T%z) #Extract the version from yml package_version=$(grep 'version:' "$package_name.yml" | awk '{print $2}' | tr -d '"' | sed 's/^v//') -#platforms=("linux/amd64" "darwin/amd64") +host_os=$(uname -s | tr '[:upper:]' '[:lower:]') +host_arch=$(uname -m) platforms=("linux/amd64") +if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then + platforms+=("darwin/amd64") +fi +#platforms=("linux/amd64") echo Building:: echo - Version $package_version @@ -40,6 +45,4 @@ do #sha256sum build/${output_name}.gz > build/${output_name}_checksum.txt done -nfpm package --config $package_name.yml --packager rpm --target build/ - ls -lah build diff --git a/scripts/update-swagger-ui.sh b/scripts/update-swagger-ui.sh index 62487bd..5904d25 100755 --- a/scripts/update-swagger-ui.sh +++ b/scripts/update-swagger-ui.sh @@ -4,7 +4,7 @@ set -euo pipefail # Usage: ./update-swagger-ui.sh [version] # Example: ./update-swagger-ui.sh v5.17.14 # If no version is provided, defaults below is used. -VERSION="${1:-v5.29.5}" +VERSION="${1:-v5.31.0}" TARGET_DIR="server/router/swagger-ui-dist" TARBALL_URL="https://github.com/swagger-api/swagger-ui/archive/refs/tags/${VERSION}.tar.gz" @@ -41,11 +41,19 @@ fi echo ">> Patching swagger-initializer.js to point at /swagger.json" -sed -i -E \ +if sed --version >/dev/null 2>&1; then + SED_INPLACE=(-i) +else + SED_INPLACE=(-i '') +fi + +append_validator=$'/url:[[:space:]]*"[^"]*swagger\\.json"[[:space:]]*,?$/a\\\n validatorUrl: null,' + +sed "${SED_INPLACE[@]}" -E \ -e 's#configUrl:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \ -e 's#url:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \ -e 's#urls:[[:space:]]*\[[^]]*\]#url: "/swagger.json"#' \ - -e '/url:[[:space:]]*"[^\"]*swagger\.json"[[:space:]]*,?$/a\ validatorUrl: null,' \ + -e "$append_validator" \ "$INDEX" -echo ">> Done. Files are in ${TARGET_DIR}" \ No newline at end of file +echo ">> Done. Files are in ${TARGET_DIR}" diff --git a/server/handler/snapshots.go b/server/handler/snapshots.go index a625ff0..49ef942 100644 --- a/server/handler/snapshots.go +++ b/server/handler/snapshots.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "vctp/components/views" "vctp/internal/report" @@ -22,7 +21,7 @@ import ( // @Failure 500 {string} string "Server error" // @Router /snapshots/hourly [get] func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_daily_", "Hourly Inventory Snapshots", views.SnapshotHourlyList) + h.renderSnapshotList(w, r, "hourly", "Hourly Inventory Snapshots", views.SnapshotHourlyList) } // SnapshotDailyList renders the daily snapshot list page. @@ -34,7 +33,7 @@ func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "Server error" // @Router /snapshots/daily [get] func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_daily_summary_", "Daily Inventory Snapshots", views.SnapshotDailyList) + h.renderSnapshotList(w, r, "daily", "Daily Inventory Snapshots", views.SnapshotDailyList) } // SnapshotMonthlyList renders the monthly snapshot list page. @@ -46,7 +45,7 @@ func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "Server error" // @Router /snapshots/monthly [get] func (h *Handler) SnapshotMonthlyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_monthly_summary_", "Monthly Inventory Snapshots", views.SnapshotMonthlyList) + h.renderSnapshotList(w, r, "monthly", "Monthly Inventory Snapshots", views.SnapshotMonthlyList) } // SnapshotReportDownload streams a snapshot table as XLSX. @@ -91,28 +90,28 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request) w.Write(reportData) } -func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, prefix string, title string, renderer func([]views.SnapshotEntry) templ.Component) { +func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, snapshotType string, title string, renderer func([]views.SnapshotEntry) templ.Component) { ctx := context.Background() - tables, err := report.ListTablesByPrefix(ctx, h.Database, prefix) + if err := report.EnsureSnapshotRegistry(ctx, h.Database); err != nil { + h.Logger.Error("Failed to ensure snapshot registry", "error", err) + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err) + return + } + records, err := report.ListSnapshots(ctx, h.Database, snapshotType) if err != nil { - h.Logger.Error("Failed to list snapshot tables", "error", err, "prefix", prefix) + h.Logger.Error("Failed to list snapshots", "error", err, "snapshot_type", snapshotType) w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err) return } - entries := make([]views.SnapshotEntry, 0, len(tables)) - for _, table := range tables { - if prefix == "inventory_daily_" && strings.HasPrefix(table, "inventory_daily_summary_") { - continue - } - label := table - if parsed, ok := report.FormatSnapshotLabel(prefix, table); ok { - label = parsed - } + entries := make([]views.SnapshotEntry, 0, len(records)) + for _, record := range records { + label := report.FormatSnapshotLabel(snapshotType, record.SnapshotTime, record.TableName) entries = append(entries, views.SnapshotEntry{ Label: label, - Link: "/api/report/snapshot?table=" + url.QueryEscape(table), + Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName), }) } diff --git a/server/handler/vmImport.go b/server/handler/vmImport.go index 738b6b1..bcabefd 100644 --- a/server/handler/vmImport.go +++ b/server/handler/vmImport.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strings" "vctp/db" queries "vctp/db/queries" models "vctp/server/models" @@ -56,6 +57,17 @@ func (h *Handler) VmImport(w http.ResponseWriter, r *http.Request) { //prettyPrint(inData) } + if strings.HasPrefix(inData.Name, "vCLS-") { + h.Logger.Info("Skipping internal vCLS VM import", "vm_name", inData.Name) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{ + "status": "OK", + "message": fmt.Sprintf("Skipped internal VM '%s'", inData.Name), + }) + return + } + ctx := context.Background() // Query Inventory table for this VM before adding it diff --git a/server/handler/vmModifyEvent.go b/server/handler/vmModifyEvent.go index 2ff2441..c7218cd 100644 --- a/server/handler/vmModifyEvent.go +++ b/server/handler/vmModifyEvent.go @@ -441,6 +441,14 @@ func (h *Handler) AddVmToInventory(evt models.CloudEventReceived, ctx context.Co } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + h.Logger.Info("Skipping internal vCLS VM", "vm_name", vmObject.Name) + if err := vc.Logout(); err != nil { + h.Logger.Error("unable to logout of vcenter", "error", err) + } + return 0, nil + } + //c.Logger.Debug("found VM") srmPlaceholder = "FALSE" // Default assumption //prettyPrint(vmObject) diff --git a/server/router/router.go b/server/router/router.go index e976547..34c3a16 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -29,6 +29,9 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st mux := http.NewServeMux() mux.Handle("/assets/", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon.ico", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon-16x16.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon-32x32.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) mux.HandleFunc("/", h.Home) mux.HandleFunc("/api/event/vm/create", h.VmCreateEvent) mux.HandleFunc("/api/event/vm/modify", h.VmModifyEvent) diff --git a/src/vctp.default b/src/vctp.default index 633406a..9e6825c 100644 --- a/src/vctp.default +++ b/src/vctp.default @@ -1 +1 @@ -CPE_OPTS='-config /etc/dtms/vctp.yml -log-level info -log-output text' +CPE_OPTS='-settings /etc/dtms/vctp.yml' diff --git a/src/vctp.service b/src/vctp.service index eb81cd7..629ce0c 100644 --- a/src/vctp.service +++ b/src/vctp.service @@ -8,6 +8,7 @@ After=network.target Type=simple EnvironmentFile=/etc/default/vctp User=vctp +Group=dtms ExecStart=/usr/bin/vctp-linux-amd64 $CPE_OPTS ExecStartPost=/usr/bin/sleep 3 Restart=always diff --git a/src/vctp.yml b/src/vctp.yml index b6ac154..64d3f20 100644 --- a/src/vctp.yml +++ b/src/vctp.yml @@ -1,25 +1,24 @@ settings: - data_location: "/var/lib/cbs" - kickstart_location: "/var/lib/cbs/ks" - database_filename: "/var/lib/cbs/cbs.db" + log_level: "info" + log_output: "text" + database_driver: "sqlite" + database_url: "/var/lib/vctp/db.sqlite3" bind_ip: - bind_port: 443 + bind_port: 9443 bind_disable_tls: false - tls_cert_filename: "/etc/dtms/cbs.crt" - tls_key_filename: "/etc/dtms/cbs.key" - tftp_root_directory: "/var/lib/tftpboot" - tftp_images_subdirectory: "images" - replacements: - omapi: - key_name: "OMAPI" - key_secret: - special_files: - ldap_groups: - ldap_bind_address: "" - ldap_base_dn: "" - ldap_trust_cert_file: "" - ldap_disable_validation: false - ldap_insecure: false - auth_token_lifespan_hours: 2 - auth_api_key: "" - \ No newline at end of file + tls_cert_filename: "/etc/dtms/vctp.crt" + tls_key_filename: "/etc/dtms/vctp.key" + vcenter_username: "" + vcenter_password: "" + vcenter_insecure: false + vcenter_event_polling_seconds: 60 + vcenter_inventory_polling_seconds: 7200 + vcenter_inventory_snapshot_seconds: 3600 + vcenter_inventory_aggregate_seconds: 86400 + hourly_snapshot_max_age_days: 60 + daily_snapshot_max_age_months: 12 + snapshot_cleanup_cron: "30 2 * * *" + tenants_to_filter: + node_charge_clusters: + srm_activeactive_vms: + vcenter_addresses: