diff --git a/.gitignore b/.gitignore index 4bf734e..8a44af5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.dylib vctp build/ +settings.yaml # Certificates *.pem diff --git a/README.md b/README.md index 8ce5c37..b8893df 100644 --- a/README.md +++ b/README.md @@ -28,27 +28,37 @@ Run `templ generate -path ./components` to generate code based on template files ## Documentation Run `swag init --exclude "pkg.mod,pkg.build,pkg.tools" -o server/router/docs` +#### Settings File +Configuration now lives in the YAML settings file. By default the service reads +`/etc/dtms/vctp.yml`, or you can override it with the `-settings` flag. + +```shell +vctp -settings /path/to/vctp.yml +``` + #### Database Configuration By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL -by setting environment variables: +by updating the settings file: -- `DB_DRIVER`: `sqlite` (default) or `postgres` -- `DB_URL`: SQLite file path/DSN or PostgreSQL DSN +- `settings.database_driver`: `sqlite` (default) or `postgres` +- `settings.database_url`: SQLite file path/DSN or PostgreSQL DSN Examples: -```shell -# SQLite (default) -DB_DRIVER=sqlite DB_URL=./db.sqlite3 +```yaml +settings: + database_driver: sqlite + database_url: ./db.sqlite3 -# PostgreSQL -DB_DRIVER=postgres DB_URL=postgres://user:pass@localhost:5432/vctp?sslmode=disable +settings: + database_driver: postgres + database_url: postgres://user:pass@localhost:5432/vctp?sslmode=disable ``` PostgreSQL migrations live in `db/migrations_postgres`, while SQLite migrations remain in `db/migrations`. #### Snapshot Retention -Hourly and daily snapshot table retention can be configured with environment variables: +Hourly and daily snapshot table retention can be configured in the settings file: -- `HOURLY_SNAPSHOT_MAX_AGE_DAYS` (default: 60) -- `DAILY_SNAPSHOT_MAX_AGE_MONTHS` (default: 12) \ No newline at end of file +- `settings.hourly_snapshot_max_age_days` (default: 60) +- `settings.daily_snapshot_max_age_months` (default: 12) diff --git a/components/core/header.templ b/components/core/header.templ index 4fa9e63..f7a9cd0 100644 --- a/components/core/header.templ +++ b/components/core/header.templ @@ -8,23 +8,25 @@ templ Header() { vCTP API + + + diff --git a/components/core/header_templ.go b/components/core/header_templ.go index 659f4b2..d261f00 100644 --- a/components/core/header_templ.go +++ b/components/core/header_templ.go @@ -31,20 +31,20 @@ func Header() templ.Component { templ_7745c5c3_Var1 = templ.NopComponent } ctx = templ.ClearChildren(ctx) - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "vCTP APIvCTP API") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\" rel=\"stylesheet\">") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/components/views/index.templ b/components/views/index.templ index 039aa71..3e6c4bb 100644 --- a/components/views/index.templ +++ b/components/views/index.templ @@ -20,10 +20,10 @@ templ Index(info BuildInfo) {
vCTP Console
-

Build Intelligence Dashboard

-

A glossy, snapshot-ready view of what is running.

+

Chargeback Intelligence Dashboard

+

Point in time snapshots of consumption.

-
+
Hourly Snapshots Daily Snapshots Monthly Snapshots diff --git a/components/views/index_templ.go b/components/views/index_templ.go index 604955d..34b411f 100644 --- a/components/views/index_templ.go +++ b/components/views/index_templ.go @@ -47,14 +47,14 @@ func Index(info BuildInfo) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "
vCTP Console

Build Intelligence Dashboard

A glossy, snapshot-ready view of what is running.

Hourly Snapshots Daily Snapshots Monthly Snapshots Swagger UI

Build Time

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "

vCTP Console

Chargeback Intelligence Dashboard

Point in time snapshots of consumption.

Hourly Snapshots Daily Snapshots Monthly Snapshots Swagger UI

Build Time

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var2 string templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(info.BuildTime) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 38, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 38, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2)) if templ_7745c5c3_Err != nil { @@ -67,7 +67,7 @@ func Index(info BuildInfo) templ.Component { var templ_7745c5c3_Var3 string templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(info.SHA1Ver) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 42, Col: 57} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 42, Col: 57} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { @@ -80,7 +80,7 @@ func Index(info BuildInfo) templ.Component { var templ_7745c5c3_Var4 string templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(info.GoVersion) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/index.templ`, Line: 46, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/index.templ`, Line: 46, Col: 59} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) if templ_7745c5c3_Err != nil { diff --git a/components/views/snapshots.templ b/components/views/snapshots.templ index c67d3a6..15d7d58 100644 --- a/components/views/snapshots.templ +++ b/components/views/snapshots.templ @@ -32,15 +32,15 @@ templ SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) {

Snapshot Library

{title}

-

{subtitle}

+

{subtitle}

Back to Dashboard
-
-

Available Exports

+
+

Available Exports 

{len(entries)} files
    diff --git a/components/views/snapshots_templ.go b/components/views/snapshots_templ.go index f4ea93c..549d398 100644 --- a/components/views/snapshots_templ.go +++ b/components/views/snapshots_templ.go @@ -140,33 +140,33 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var5 string templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(title) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 34, Col: 49} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 34, Col: 49} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

    ") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "

    ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(subtitle) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 35, Col: 51} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 35, Col: 55} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "

Back to Dashboard

Available Exports

") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "

Back to Dashboard

Available Exports 

") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 string templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(len(entries)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 44, Col: 83} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 44, Col: 83} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) if templ_7745c5c3_Err != nil { @@ -184,7 +184,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var8 string templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(entry.Label) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 49, Col: 71} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 49, Col: 71} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { @@ -197,7 +197,7 @@ func SnapshotListPage(title string, subtitle string, entries []SnapshotEntry) te var templ_7745c5c3_Var9 templ.SafeURL templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(entry.Link) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `components/views/snapshots.templ`, Line: 50, Col: 45} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `views/snapshots.templ`, Line: 50, Col: 45} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { diff --git a/db/migrations/20250115094500_snapshot_registry.sql b/db/migrations/20250115094500_snapshot_registry.sql new file mode 100644 index 0000000..f54d821 --- /dev/null +++ b/db/migrations/20250115094500_snapshot_registry.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE snapshot_registry; +-- +goose StatementEnd diff --git a/db/migrations_postgres/20250115094500_snapshot_registry.sql b/db/migrations_postgres/20250115094500_snapshot_registry.sql new file mode 100644 index 0000000..69fa174 --- /dev/null +++ b/db/migrations_postgres/20250115094500_snapshot_registry.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id BIGSERIAL PRIMARY KEY, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE snapshot_registry; +-- +goose StatementEnd diff --git a/db/schema.sql b/db/schema.sql index 33f91ca..a646637 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -66,3 +66,10 @@ CREATE TABLE IF NOT EXISTS inventory_history ( "PreviousResourcePool" TEXT, "PreviousProvisionedDisk" REAL ); + +CREATE TABLE IF NOT EXISTS snapshot_registry ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "snapshot_type" TEXT NOT NULL, + "table_name" TEXT NOT NULL UNIQUE, + "snapshot_time" INTEGER NOT NULL +); diff --git a/dist/dist.go b/dist/dist.go index 02c066e..01e24de 100644 --- a/dist/dist.go +++ b/dist/dist.go @@ -4,5 +4,5 @@ import ( "embed" ) -//go:embed all:assets +//go:embed all:assets favicon.ico favicon-16x16.png favicon-32x32.png var AssetsDir embed.FS diff --git a/dist/favicon-16x16.png b/dist/favicon-16x16.png new file mode 100644 index 0000000..e0f8e68 Binary files /dev/null and b/dist/favicon-16x16.png differ diff --git a/dist/favicon-32x32.png b/dist/favicon-32x32.png new file mode 100644 index 0000000..0931913 Binary files /dev/null and b/dist/favicon-32x32.png differ diff --git a/dist/favicon.ico b/dist/favicon.ico new file mode 100644 index 0000000..736d809 Binary files /dev/null and b/dist/favicon.ico differ diff --git a/e2e/e2e_test.go.txt b/e2e/e2e_test.go.txt index 69d8618..65badea 100644 --- a/e2e/e2e_test.go.txt +++ b/e2e/e2e_test.go.txt @@ -95,14 +95,33 @@ func beforeAll() { func startApp() error { port := getPort() - app = exec.Command("go", "run", "main.go") + settingsPath := "./test-settings.yml" + settingsBody := fmt.Sprintf(`settings: + log_level: "debug" + log_output: "text" + database_driver: "sqlite" + database_url: "./test-db.sqlite3" + bind_ip: "127.0.0.1" + bind_port: %d + bind_disable_tls: true + tls_cert_filename: + tls_key_filename: + vcenter_username: "test" + vcenter_password: "test" + vcenter_insecure: true + vcenter_event_polling_seconds: 60 + vcenter_inventory_polling_seconds: 7200 + vcenter_inventory_snapshot_seconds: 3600 + vcenter_inventory_aggregate_seconds: 86400 + hourly_snapshot_max_age_days: 1 + daily_snapshot_max_age_months: 1 +`, port) + if err := os.WriteFile("../"+settingsPath, []byte(settingsBody), 0o600); err != nil { + return err + } + app = exec.Command("go", "run", "main.go", "-settings", settingsPath) app.Dir = "../" - app.Env = append( - os.Environ(), - "DB_URL=./test-db.sqlite3", - fmt.Sprintf("PORT=%d", port), - "LOG_LEVEL=DEBUG", - ) + app.Env = os.Environ() var err error baseUrL, err = url.Parse(fmt.Sprintf("http://localhost:%d", port)) @@ -188,6 +207,9 @@ func afterAll() { if err := os.Remove("../test-db.sqlite3"); err != nil { log.Fatalf("could not remove test-db.sqlite3: %v", err) } + if err := os.Remove("../test-settings.yml"); err != nil { + log.Fatalf("could not remove test-settings.yml: %v", err) + } } // beforeEach creates a new context and page for each test, diff --git a/go.mod b/go.mod index a385cc8..69bc77b 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,42 @@ module vctp -go 1.25.0 +go 1.25.5 require ( github.com/a-h/templ v0.3.977 github.com/go-co-op/gocron/v2 v2.19.0 github.com/jackc/pgx/v5 v5.8.0 github.com/jmoiron/sqlx v1.4.0 - github.com/joho/godotenv v1.5.1 github.com/pressly/goose/v3 v3.26.0 + github.com/swaggo/swag v1.16.6 github.com/vmware/govmomi v0.52.0 github.com/xuri/excelize/v2 v2.10.0 gopkg.in/yaml.v2 v2.4.0 - modernc.org/sqlite v1.43.0 + modernc.org/sqlite v1.44.0 ) require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.6 // indirect - github.com/richardlehane/msoleps v1.0.5 // indirect + github.com/richardlehane/msoleps v1.0.6 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/tiendc/go-deepcopy v1.7.2 // indirect @@ -36,10 +45,12 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.32.0 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect + golang.org/x/tools v0.41.0 // indirect modernc.org/libc v1.67.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 4496990..5c008eb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg= github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -9,6 +16,16 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-co-op/gocron/v2 v2.19.0 h1:OKf2y6LXPs/BgBI2fl8PxUpNAI1DA9Mg+hSeGOS38OU= github.com/go-co-op/gocron/v2 v2.19.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -30,16 +47,23 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -48,6 +72,7 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= @@ -58,6 +83,8 @@ github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= github.com/richardlehane/msoleps v1.0.5 h1:kNlmACZuwC8ZWPLoJtD+HtZOsKJgYn7gXgUIcRB7dbo= github.com/richardlehane/msoleps v1.0.5/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= +github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -66,9 +93,12 @@ github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= +github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44= github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= github.com/vmware/govmomi v0.52.0 h1:JyxQ1IQdllrY7PJbv2am9mRsv3p9xWlIQ66bv+XnyLw= @@ -91,23 +121,34 @@ golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= @@ -134,6 +175,8 @@ modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.43.0 h1:8YqiFx3G1VhHTXO2Q00bl1Wz9KhS9Q5okwfp9Y97VnA= modernc.org/sqlite v1.43.0/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= +modernc.org/sqlite v1.44.0 h1:YjCKJnzZde2mLVy0cMKTSL4PxCmbIguOq9lGp8ZvGOc= +modernc.org/sqlite v1.44.0/go.mod h1:2Dq41ir5/qri7QJJJKNZcP4UF7TsX/KNeykYgPDtGhE= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/report/snapshots.go b/internal/report/snapshots.go index 23bc4c8..a1dbaf0 100644 --- a/internal/report/snapshots.go +++ b/internal/report/snapshots.go @@ -14,6 +14,12 @@ import ( "github.com/xuri/excelize/v2" ) +type SnapshotRecord struct { + TableName string + SnapshotTime time.Time + SnapshotType string +} + func ListTablesByPrefix(ctx context.Context, database db.Database, prefix string) ([]string, error) { dbConn := database.DB() driver := strings.ToLower(dbConn.DriverName()) @@ -59,26 +65,139 @@ ORDER BY tablename DESC return tables, rows.Err() } -func FormatSnapshotLabel(prefix string, tableName string) (string, bool) { - if !strings.HasPrefix(tableName, prefix) { - return "", false +func EnsureSnapshotRegistry(ctx context.Context, database db.Database) error { + dbConn := database.DB() + driver := strings.ToLower(dbConn.DriverName()) + switch driver { + case "sqlite": + _, err := dbConn.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +) +`) + return err + case "pgx", "postgres": + _, err := dbConn.ExecContext(ctx, ` +CREATE TABLE IF NOT EXISTS snapshot_registry ( + id BIGSERIAL PRIMARY KEY, + snapshot_type TEXT NOT NULL, + table_name TEXT NOT NULL UNIQUE, + snapshot_time BIGINT NOT NULL +) +`) + return err + default: + return fmt.Errorf("unsupported driver for snapshot registry: %s", driver) } - suffix := strings.TrimPrefix(tableName, prefix) - switch prefix { - case "inventory_daily_": - if t, err := time.Parse("20060102", suffix); err == nil { - return t.Format("2006-01-02"), true - } - case "inventory_daily_summary_": - if t, err := time.Parse("20060102", suffix); err == nil { - return t.Format("2006-01-02"), true - } - case "inventory_monthly_summary_": - if t, err := time.Parse("200601", suffix); err == nil { - return t.Format("2006-01"), true - } +} + +func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error { + if snapshotType == "" || tableName == "" { + return fmt.Errorf("snapshot type or table name is empty") + } + dbConn := database.DB() + driver := strings.ToLower(dbConn.DriverName()) + switch driver { + case "sqlite": + _, err := dbConn.ExecContext(ctx, ` +INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time) +VALUES (?, ?, ?) +`, snapshotType, tableName, snapshotTime.Unix()) + return err + case "pgx", "postgres": + _, err := dbConn.ExecContext(ctx, ` +INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time) +VALUES ($1, $2, $3) +ON CONFLICT (table_name) DO NOTHING +`, snapshotType, tableName, snapshotTime.Unix()) + return err + default: + return fmt.Errorf("unsupported driver for snapshot registry: %s", driver) + } +} + +func DeleteSnapshotRecord(ctx context.Context, database db.Database, tableName string) error { + if tableName == "" { + return nil + } + dbConn := database.DB() + driver := strings.ToLower(dbConn.DriverName()) + switch driver { + case "sqlite": + _, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry WHERE table_name = ?`, tableName) + return err + case "pgx", "postgres": + _, err := dbConn.ExecContext(ctx, `DELETE FROM snapshot_registry WHERE table_name = $1`, tableName) + return err + default: + return fmt.Errorf("unsupported driver for snapshot registry: %s", driver) + } +} + +func ListSnapshots(ctx context.Context, database db.Database, snapshotType string) ([]SnapshotRecord, error) { + dbConn := database.DB() + driver := strings.ToLower(dbConn.DriverName()) + + var rows *sqlx.Rows + var err error + + switch driver { + case "sqlite": + rows, err = dbConn.QueryxContext(ctx, ` +SELECT table_name, snapshot_time, snapshot_type +FROM snapshot_registry +WHERE snapshot_type = ? +ORDER BY snapshot_time DESC, table_name DESC +`, snapshotType) + case "pgx", "postgres": + rows, err = dbConn.QueryxContext(ctx, ` +SELECT table_name, snapshot_time, snapshot_type +FROM snapshot_registry +WHERE snapshot_type = $1 +ORDER BY snapshot_time DESC, table_name DESC +`, snapshotType) + default: + return nil, fmt.Errorf("unsupported driver for listing snapshots: %s", driver) + } + + if err != nil { + return nil, err + } + defer rows.Close() + + records := make([]SnapshotRecord, 0) + for rows.Next() { + var ( + tableName string + snapshotTime int64 + recordType string + ) + if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil { + return nil, err + } + records = append(records, SnapshotRecord{ + TableName: tableName, + SnapshotTime: time.Unix(snapshotTime, 0), + SnapshotType: recordType, + }) + } + return records, rows.Err() +} + +func FormatSnapshotLabel(snapshotType string, snapshotTime time.Time, tableName string) string { + switch snapshotType { + case "hourly": + return snapshotTime.Format("2006-01-02 15:00") + case "daily": + return snapshotTime.Format("2006-01-02") + case "monthly": + return snapshotTime.Format("2006-01") + default: + return tableName } - return "", false } func CreateTableReport(logger *slog.Logger, Database db.Database, ctx context.Context, tableName string) ([]byte, error) { diff --git a/internal/settings/settings.go b/internal/settings/settings.go index abbafd1..e930918 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "vctp/internal/utils" "gopkg.in/yaml.v2" @@ -19,10 +20,29 @@ type Settings struct { // SettingsYML struct holds various runtime data that is too cumbersome to specify via command line, eg replacement properties type SettingsYML struct { Settings struct { - TenantsToFilter []string `yaml:"tenants_to_filter"` - NodeChargeClusters []string `yaml:"node_charge_clusters"` - SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"` - VcenterAddresses []string `yaml:"vcenter_addresses"` + LogLevel string `yaml:"log_level"` + LogOutput string `yaml:"log_output"` + DatabaseDriver string `yaml:"database_driver"` + DatabaseURL string `yaml:"database_url"` + BindIP string `yaml:"bind_ip"` + BindPort int `yaml:"bind_port"` + BindDisableTLS bool `yaml:"bind_disable_tls"` + TLSCertFilename string `yaml:"tls_cert_filename"` + TLSKeyFilename string `yaml:"tls_key_filename"` + VcenterUsername string `yaml:"vcenter_username"` + VcenterPassword string `yaml:"vcenter_password"` + VcenterInsecure bool `yaml:"vcenter_insecure"` + VcenterEventPollingSeconds int `yaml:"vcenter_event_polling_seconds"` + VcenterInventoryPollingSeconds int `yaml:"vcenter_inventory_polling_seconds"` + VcenterInventorySnapshotSeconds int `yaml:"vcenter_inventory_snapshot_seconds"` + VcenterInventoryAggregateSeconds int `yaml:"vcenter_inventory_aggregate_seconds"` + HourlySnapshotMaxAgeDays int `yaml:"hourly_snapshot_max_age_days"` + DailySnapshotMaxAgeMonths int `yaml:"daily_snapshot_max_age_months"` + SnapshotCleanupCron string `yaml:"snapshot_cleanup_cron"` + TenantsToFilter []string `yaml:"tenants_to_filter"` + NodeChargeClusters []string `yaml:"node_charge_clusters"` + SrmActiveActiveVms []string `yaml:"srm_activeactive_vms"` + VcenterAddresses []string `yaml:"vcenter_addresses"` } `yaml:"settings"` } @@ -65,3 +85,49 @@ func (s *Settings) ReadYMLSettings() error { return nil } + +func (s *Settings) WriteYMLSettings() error { + if s.Values == nil { + return errors.New("settings are not loaded") + } + if len(s.SettingsPath) == 0 { + return errors.New("settings file path not specified") + } + + data, err := yaml.Marshal(s.Values) + if err != nil { + return fmt.Errorf("unable to encode settings file: %w", err) + } + + mode := os.FileMode(0o644) + if info, err := os.Stat(s.SettingsPath); err == nil { + mode = info.Mode().Perm() + } + + dir := filepath.Dir(s.SettingsPath) + tmp, err := os.CreateTemp(dir, "vctp-settings-*.yml") + if err != nil { + return fmt.Errorf("unable to create temp settings file: %w", err) + } + tmpName := tmp.Name() + defer func() { + _ = os.Remove(tmpName) + }() + + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("unable to write temp settings file: %w", err) + } + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + return fmt.Errorf("unable to set temp settings permissions: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("unable to close temp settings file: %w", err) + } + if err := os.Rename(tmpName, s.SettingsPath); err != nil { + return fmt.Errorf("unable to replace settings file: %w", err) + } + + return nil +} diff --git a/internal/tasks/inventorySnapshots.go b/internal/tasks/inventorySnapshots.go index 0201c44..6d92bb6 100644 --- a/internal/tasks/inventorySnapshots.go +++ b/internal/tasks/inventorySnapshots.go @@ -5,8 +5,6 @@ import ( "database/sql" "fmt" "log/slog" - "os" - "strconv" "strings" "time" "vctp/db/queries" @@ -33,8 +31,8 @@ type inventorySnapshotRow struct { Cluster sql.NullString Folder sql.NullString ProvisionedDisk sql.NullFloat64 - InitialVcpus sql.NullInt64 - InitialRam sql.NullInt64 + VcpuCount sql.NullInt64 + RamGB sql.NullInt64 IsTemplate string PoweredOn string SrmPlaceholder string @@ -46,7 +44,7 @@ type inventorySnapshotRow struct { // RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table. func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) error { startTime := time.Now() - tableName, err := dailyInventoryTableName(startTime) + tableName, err := hourlyInventoryTableName(startTime) if err != nil { return err } @@ -55,6 +53,12 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo if err := ensureDailyInventoryTable(ctx, dbConn, tableName); err != nil { return err } + if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { + return err + } + if err := report.RegisterSnapshot(ctx, c.Database, "hourly", tableName, startTime); err != nil { + c.Logger.Warn("failed to register hourly snapshot", "error", err, "table", tableName) + } // reload settings in case vcenter list has changed c.Settings.ReadYMLSettings() @@ -62,7 +66,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo for _, url := range c.Settings.Values.Settings.VcenterAddresses { c.Logger.Debug("connecting to vcenter for hourly snapshot", "url", url) vc := vcenter.New(c.Logger, c.VcCreds) - vc.Login(url) + if err := vc.Login(url); err != nil { + c.Logger.Error("unable to connect to vcenter for hourly snapshot", "error", err, "url", url) + continue + } vcVms, err := vc.GetAllVmReferences() if err != nil { @@ -70,6 +77,10 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo vc.Logout() continue } + canDetectMissing := len(vcVms) > 0 + if !canDetectMissing { + c.Logger.Warn("no VMs returned from vcenter; skipping missing VM detection", "url", url) + } inventoryRows, err := c.Database.Queries().GetInventoryByVcenter(ctx, url) if err != nil { @@ -116,8 +127,8 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo presentSnapshots[vm.Reference().Value] = row totals.VmCount++ - totals.VcpuTotal += nullInt64ToInt(row.InitialVcpus) - totals.RamTotal += nullInt64ToInt(row.InitialRam) + totals.VcpuTotal += nullInt64ToInt(row.VcpuCount) + totals.RamTotal += nullInt64ToInt(row.RamGB) totals.DiskTotal += nullFloat64ToFloat(row.ProvisionedDisk) } @@ -127,7 +138,15 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo } } + if !canDetectMissing { + vc.Logout() + continue + } + for _, inv := range inventoryRows { + if strings.HasPrefix(inv.Name, "vCLS-") { + continue + } vmID := inv.VmId.String if vmID != "" { if _, ok := presentSnapshots[vmID]; ok { @@ -137,6 +156,17 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo row := snapshotFromInventory(inv, startTime) row.IsPresent = "FALSE" + if !row.DeletionTime.Valid { + deletionTime := startTime.Unix() + row.DeletionTime = sql.NullInt64{Int64: deletionTime, Valid: true} + if err := c.Database.Queries().InventoryMarkDeleted(ctx, queries.InventoryMarkDeletedParams{ + DeletionTime: row.DeletionTime, + VmId: inv.VmId, + DatacenterName: inv.Datacenter, + }); err != nil { + c.Logger.Warn("failed to mark inventory record deleted", "error", err, "vm_id", row.VmId.String) + } + } if err := insertDailyInventoryRow(ctx, dbConn, tableName, row); err != nil { c.Logger.Error("failed to insert missing VM snapshot", "error", err, "vm_id", row.VmId.String) } @@ -148,7 +178,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo "vcenter", url, "vm_count", totals.VmCount, "vcpu_total", totals.VcpuTotal, - "ram_total_mb", totals.RamTotal, + "ram_total_gb", totals.RamTotal, "disk_total_gb", totals.DiskTotal, ) } @@ -160,7 +190,7 @@ func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Lo // RunVcenterDailyAggregate summarizes hourly snapshots into a daily summary table. func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Logger) error { targetTime := time.Now().Add(-time.Minute) - sourceTable, err := dailyInventoryTableName(targetTime) + sourceTable, err := hourlyInventoryTableName(targetTime) if err != nil { return err } @@ -173,6 +203,9 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo if err := ensureDailySummaryTable(ctx, dbConn, summaryTable); err != nil { return err } + if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { + return err + } currentTotals, err := snapshotTotalsForTable(ctx, dbConn, sourceTable) if err != nil { @@ -182,12 +215,12 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo "table", sourceTable, "vm_count", currentTotals.VmCount, "vcpu_total", currentTotals.VcpuTotal, - "ram_total_mb", currentTotals.RamTotal, + "ram_total_gb", currentTotals.RamTotal, "disk_total_gb", currentTotals.DiskTotal, ) } - prevTable, _ := dailyInventoryTableName(targetTime.AddDate(0, 0, -1)) + prevTable, _ := hourlyInventoryTableName(targetTime.AddDate(0, 0, -1)) if prevTable != "" && tableExists(ctx, dbConn, prevTable) { prevTotals, err := snapshotTotalsForTable(ctx, dbConn, prevTable) if err != nil { @@ -198,7 +231,7 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo "previous_table", prevTable, "vm_delta", currentTotals.VmCount-prevTotals.VmCount, "vcpu_delta", currentTotals.VcpuTotal-prevTotals.VcpuTotal, - "ram_delta_mb", currentTotals.RamTotal-prevTotals.RamTotal, + "ram_delta_gb", currentTotals.RamTotal-prevTotals.RamTotal, "disk_delta_gb", currentTotals.DiskTotal-prevTotals.DiskTotal, ) } @@ -207,19 +240,19 @@ func (c *CronTask) RunVcenterDailyAggregate(ctx context.Context, logger *slog.Lo insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "SamplesPresent", "AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "SamplesPresent", "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct" ) SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "SamplesPresent", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam", - AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "IsPresent" = 'TRUE' AND "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", @@ -232,14 +265,17 @@ SELECT FROM %s GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, summaryTable, sourceTable) if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate daily inventory", "error", err, "source_table", sourceTable) return err } + if err := report.RegisterSnapshot(ctx, c.Database, "daily", summaryTable, targetTime); err != nil { + c.Logger.Warn("failed to register daily snapshot", "error", err, "table", summaryTable) + } c.Logger.Debug("Finished daily inventory aggregation", "source_table", sourceTable, "summary_table", summaryTable) return nil @@ -251,7 +287,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. firstOfThisMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) targetMonth := firstOfThisMonth.AddDate(0, -1, 0) - monthPrefix := fmt.Sprintf("inventory_daily_%s", targetMonth.Format("200601")) + monthPrefix := fmt.Sprintf("inventory_hourly_%s", targetMonth.Format("200601")) dailyTables, err := report.ListTablesByPrefix(ctx, c.Database, monthPrefix) if err != nil { return err @@ -269,11 +305,14 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. if err := ensureMonthlySummaryTable(ctx, dbConn, monthlyTable); err != nil { return err } + if err := report.EnsureSnapshotRegistry(ctx, c.Database); err != nil { + return err + } unionQuery := buildUnionQuery(dailyTables, []string{ `"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`, `"DeletionTime"`, `"ResourcePool"`, `"VmType"`, `"Datacenter"`, `"Cluster"`, `"Folder"`, - `"ProvisionedDisk"`, `"InitialVcpus"`, `"InitialRam"`, `"IsTemplate"`, `"PoweredOn"`, + `"ProvisionedDisk"`, `"VcpuCount"`, `"RamGB"`, `"IsTemplate"`, `"PoweredOn"`, `"SrmPlaceholder"`, `"VmUuid"`, `"IsPresent"`, }) if strings.TrimSpace(unionQuery) == "" { @@ -288,7 +327,7 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. "month", targetMonth.Format("2006-01"), "vm_count", monthlyTotals.VmCount, "vcpu_total", monthlyTotals.VcpuTotal, - "ram_total_mb", monthlyTotals.RamTotal, + "ram_total_gb", monthlyTotals.RamTotal, "disk_total_gb", monthlyTotals.DiskTotal, ) } @@ -296,18 +335,18 @@ func (c *CronTask) RunVcenterMonthlyAggregate(ctx context.Context, logger *slog. insertQuery := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - "AvgVcpus", "AvgRam", "AvgDisk", "AvgIsPresent", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + "AvgVcpuCount", "AvgRamGB", "AvgProvisionedDisk", "AvgIsPresent", "PoolTinPct", "PoolBronzePct", "PoolSilverPct", "PoolGoldPct" ) SELECT "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", - AVG(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" END) AS "AvgVcpus", - AVG(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" END) AS "AvgRam", - AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgDisk", + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", + AVG(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" END) AS "AvgVcpuCount", + AVG(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" END) AS "AvgRamGB", + AVG(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" END) AS "AvgProvisionedDisk", AVG(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END) AS "AvgIsPresent", 100.0 * SUM(CASE WHEN "IsPresent" = 'TRUE' AND LOWER("ResourcePool") = 'tin' THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN "IsPresent" = 'TRUE' THEN 1 ELSE 0 END), 0) AS "PoolTinPct", @@ -322,14 +361,17 @@ FROM ( ) snapshots GROUP BY "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid"; `, monthlyTable, unionQuery) if _, err := dbConn.ExecContext(ctx, insertQuery); err != nil { c.Logger.Error("failed to aggregate monthly inventory", "error", err, "month", targetMonth.Format("2006-01")) return err } + if err := report.RegisterSnapshot(ctx, c.Database, "monthly", monthlyTable, targetMonth); err != nil { + c.Logger.Warn("failed to register monthly snapshot", "error", err, "table", monthlyTable) + } c.Logger.Debug("Finished monthly inventory aggregation", "summary_table", monthlyTable) return nil @@ -338,15 +380,15 @@ GROUP BY // RunSnapshotCleanup drops hourly and daily snapshot tables older than retention. func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) error { now := time.Now() - hourlyMaxDays := getEnvInt("HOURLY_SNAPSHOT_MAX_AGE_DAYS", 60) - dailyMaxMonths := getEnvInt("DAILY_SNAPSHOT_MAX_AGE_MONTHS", 12) + hourlyMaxDays := intWithDefault(c.Settings.Values.Settings.HourlySnapshotMaxAgeDays, 60) + dailyMaxMonths := intWithDefault(c.Settings.Values.Settings.DailySnapshotMaxAgeMonths, 12) hourlyCutoff := now.AddDate(0, 0, -hourlyMaxDays) dailyCutoff := now.AddDate(0, -dailyMaxMonths, 0) dbConn := c.Database.DB() - hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_daily_") + hourlyTables, err := report.ListTablesByPrefix(ctx, c.Database, "inventory_hourly_") if err != nil { return err } @@ -356,7 +398,7 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) if strings.HasPrefix(table, "inventory_daily_summary_") { continue } - tableDate, ok := parseSnapshotDate(table, "inventory_daily_", "20060102") + tableDate, ok := parseSnapshotDate(table, "inventory_hourly_", "2006010215") if !ok { continue } @@ -365,6 +407,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) c.Logger.Error("failed to drop hourly snapshot table", "error", err, "table", table) } else { removedHourly++ + if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil { + c.Logger.Warn("failed to remove hourly snapshot registry entry", "error", err, "table", table) + } } } } @@ -384,6 +429,9 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) c.Logger.Error("failed to drop daily snapshot table", "error", err, "table", table) } else { removedDaily++ + if err := report.DeleteSnapshotRecord(ctx, c.Database, table); err != nil { + c.Logger.Warn("failed to remove daily snapshot registry entry", "error", err, "table", table) + } } } } @@ -397,8 +445,8 @@ func (c *CronTask) RunSnapshotCleanup(ctx context.Context, logger *slog.Logger) return nil } -func dailyInventoryTableName(t time.Time) (string, error) { - return safeTableName(fmt.Sprintf("inventory_daily_%s", t.Format("20060102"))) +func hourlyInventoryTableName(t time.Time) (string, error) { + return safeTableName(fmt.Sprintf("inventory_hourly_%s", t.Format("2006010215"))) } func dailySummaryTableName(t time.Time) (string, error) { @@ -435,8 +483,8 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, @@ -445,8 +493,14 @@ func ensureDailyInventoryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "IsPresent" TEXT NOT NULL );`, tableName) - _, err := dbConn.ExecContext(ctx, ddl) - return err + if _, err := dbConn.ExecContext(ctx, ddl); err != nil { + return err + } + + return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ + {Name: "VcpuCount", Type: "BIGINT"}, + {Name: "RamGB", Type: "BIGINT"}, + }) } func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName string) error { @@ -465,16 +519,16 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, "VmUuid" TEXT, "SamplesPresent" BIGINT NOT NULL, - "AvgVcpus" REAL, - "AvgRam" REAL, - "AvgDisk" REAL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, "AvgIsPresent" REAL, "PoolTinPct" REAL, "PoolBronzePct" REAL, @@ -487,9 +541,9 @@ func ensureDailySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName str } return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgVcpus", Type: "REAL"}, - {Name: "AvgRam", Type: "REAL"}, - {Name: "AvgDisk", Type: "REAL"}, + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, {Name: "AvgIsPresent", Type: "REAL"}, {Name: "PoolTinPct", Type: "REAL"}, {Name: "PoolBronzePct", Type: "REAL"}, @@ -514,15 +568,15 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s "Cluster" TEXT, "Folder" TEXT, "ProvisionedDisk" REAL, - "InitialVcpus" BIGINT, - "InitialRam" BIGINT, + "VcpuCount" BIGINT, + "RamGB" BIGINT, "IsTemplate" TEXT, "PoweredOn" TEXT, "SrmPlaceholder" TEXT, "VmUuid" TEXT, - "AvgVcpus" REAL, - "AvgRam" REAL, - "AvgDisk" REAL, + "AvgVcpuCount" REAL, + "AvgRamGB" REAL, + "AvgProvisionedDisk" REAL, "AvgIsPresent" REAL, "PoolTinPct" REAL, "PoolBronzePct" REAL, @@ -535,7 +589,10 @@ func ensureMonthlySummaryTable(ctx context.Context, dbConn *sqlx.DB, tableName s } return ensureSnapshotColumns(ctx, dbConn, tableName, []columnDef{ - {Name: "AvgDisk", Type: "REAL"}, + {Name: "AvgVcpuCount", Type: "REAL"}, + {Name: "AvgRamGB", Type: "REAL"}, + {Name: "AvgProvisionedDisk", Type: "REAL"}, + {Name: "AvgIsPresent", Type: "REAL"}, {Name: "PoolTinPct", Type: "REAL"}, {Name: "PoolBronzePct", Type: "REAL"}, {Name: "PoolSilverPct", Type: "REAL"}, @@ -622,8 +679,8 @@ func snapshotTotalsForTable(ctx context.Context, dbConn *sqlx.DB, table string) query := fmt.Sprintf(` SELECT COUNT(DISTINCT "VmId") AS vm_count, - COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total, - COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total, + COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total, + COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total, COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total FROM %s WHERE "IsPresent" = 'TRUE' @@ -640,8 +697,8 @@ func snapshotTotalsForUnion(ctx context.Context, dbConn *sqlx.DB, unionQuery str query := fmt.Sprintf(` SELECT COUNT(DISTINCT "VmId") AS vm_count, - COALESCE(SUM(CASE WHEN "InitialVcpus" IS NOT NULL THEN "InitialVcpus" ELSE 0 END), 0) AS vcpu_total, - COALESCE(SUM(CASE WHEN "InitialRam" IS NOT NULL THEN "InitialRam" ELSE 0 END), 0) AS ram_total, + COALESCE(SUM(CASE WHEN "VcpuCount" IS NOT NULL THEN "VcpuCount" ELSE 0 END), 0) AS vcpu_total, + COALESCE(SUM(CASE WHEN "RamGB" IS NOT NULL THEN "RamGB" ELSE 0 END), 0) AS ram_total, COALESCE(SUM(CASE WHEN "ProvisionedDisk" IS NOT NULL THEN "ProvisionedDisk" ELSE 0 END), 0) AS disk_total FROM ( %s @@ -694,13 +751,8 @@ func nullFloat64ToFloat(value sql.NullFloat64) float64 { return 0 } -func getEnvInt(key string, fallback int) int { - raw := strings.TrimSpace(os.Getenv(key)) - if raw == "" { - return fallback - } - value, err := strconv.Atoi(raw) - if err != nil || value < 0 { +func intWithDefault(value int, fallback int) int { + if value <= 0 { return fallback } return value @@ -731,8 +783,8 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi if !vmObject.Config.CreateDate.IsZero() { row.CreationTime = sql.NullInt64{Int64: vmObject.Config.CreateDate.Unix(), Valid: true} } - row.InitialVcpus = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0} - row.InitialRam = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB), Valid: vmObject.Config.Hardware.MemoryMB > 0} + row.VcpuCount = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.NumCPU), Valid: vmObject.Config.Hardware.NumCPU > 0} + row.RamGB = sql.NullInt64{Int64: int64(vmObject.Config.Hardware.MemoryMB) / 1024, Valid: vmObject.Config.Hardware.MemoryMB > 0} totalDiskBytes := int64(0) for _, device := range vmObject.Config.Hardware.Device { @@ -774,11 +826,11 @@ func snapshotFromVM(vmObject *mo.VirtualMachine, vc *vcenter.Vcenter, snapshotTi if !row.ProvisionedDisk.Valid { row.ProvisionedDisk = inv.ProvisionedDisk } - if !row.InitialVcpus.Valid { - row.InitialVcpus = inv.InitialVcpus + if !row.VcpuCount.Valid { + row.VcpuCount = inv.InitialVcpus } - if !row.InitialRam.Valid { - row.InitialRam = inv.InitialRam + if !row.RamGB.Valid && inv.InitialRam.Valid { + row.RamGB = sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Int64 > 0} } if row.IsTemplate == "" { row.IsTemplate = boolStringFromInterface(inv.IsTemplate) @@ -837,8 +889,8 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent Cluster: inv.Cluster, Folder: inv.Folder, ProvisionedDisk: inv.ProvisionedDisk, - InitialVcpus: inv.InitialVcpus, - InitialRam: inv.InitialRam, + VcpuCount: inv.InitialVcpus, + RamGB: sql.NullInt64{Int64: inv.InitialRam.Int64 / 1024, Valid: inv.InitialRam.Valid && inv.InitialRam.Int64 > 0}, IsTemplate: boolStringFromInterface(inv.IsTemplate), PoweredOn: boolStringFromInterface(inv.PoweredOn), SrmPlaceholder: boolStringFromInterface(inv.SrmPlaceholder), @@ -851,8 +903,8 @@ func insertDailyInventoryRow(ctx context.Context, dbConn *sqlx.DB, tableName str query := fmt.Sprintf(` INSERT INTO %s ( "InventoryId", "Name", "Vcenter", "VmId", "EventKey", "CloudId", "CreationTime", "DeletionTime", - "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "InitialVcpus", - "InitialRam", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent" + "ResourcePool", "VmType", "Datacenter", "Cluster", "Folder", "ProvisionedDisk", "VcpuCount", + "RamGB", "IsTemplate", "PoweredOn", "SrmPlaceholder", "VmUuid", "SnapshotTime", "IsPresent" ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); `, tableName) @@ -874,8 +926,8 @@ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); row.Cluster, row.Folder, row.ProvisionedDisk, - row.InitialVcpus, - row.InitialRam, + row.VcpuCount, + row.RamGB, row.IsTemplate, row.PoweredOn, row.SrmPlaceholder, diff --git a/internal/tasks/monitorVcenter.go b/internal/tasks/monitorVcenter.go index 0af5f6e..1710458 100644 --- a/internal/tasks/monitorVcenter.go +++ b/internal/tasks/monitorVcenter.go @@ -262,6 +262,11 @@ func (c *CronTask) AddVmToInventory(vmObject *mo.VirtualMachine, vc *vcenter.Vce return errors.New("can't process empty vm object") } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + c.Logger.Debug("Skipping internal vCLS VM", "vm_name", vmObject.Name) + return nil + } + c.Logger.Debug("found VM") /* diff --git a/internal/tasks/processEvents.go b/internal/tasks/processEvents.go index dd45035..377e6b8 100644 --- a/internal/tasks/processEvents.go +++ b/internal/tasks/processEvents.go @@ -83,6 +83,14 @@ func (c *CronTask) RunVmCheck(ctx context.Context, logger *slog.Logger) error { continue } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + c.Logger.Info("Skipping internal vCLS VM event", "vm_name", vmObject.Name) + if err := c.Database.Queries().UpdateEventsProcessed(ctx, evt.Eid); err != nil { + c.Logger.Error("Unable to mark vCLS event as processed", "event_id", evt.Eid, "error", err) + } + continue + } + //c.Logger.Debug("found VM") srmPlaceholder = "FALSE" // Default assumption //prettyPrint(vmObject) diff --git a/internal/vcenter/vcenter.go b/internal/vcenter/vcenter.go index e5a7796..8a641fb 100644 --- a/internal/vcenter/vcenter.go +++ b/internal/vcenter/vcenter.go @@ -6,7 +6,6 @@ import ( "log" "log/slog" "net/url" - "os" "path" "strings" @@ -30,6 +29,7 @@ type Vcenter struct { type VcenterLogin struct { Username string Password string + Insecure bool } type VmProperties struct { @@ -51,13 +51,6 @@ func New(logger *slog.Logger, creds *VcenterLogin) *Vcenter { } func (v *Vcenter) Login(vUrl string) error { - var insecure bool - - // TODO - fix this - insecureString := os.Getenv("VCENTER_INSECURE") - //username := os.Getenv("VCENTER_USERNAME") - //password := os.Getenv("VCENTER_PASSWORD") - // Connect to vCenter u, err := soap.ParseURL(vUrl) if err != nil { @@ -74,11 +67,7 @@ func (v *Vcenter) Login(vUrl string) error { } */ - if insecureString == "true" { - insecure = true - } - - c, err := govmomi.NewClient(v.ctx, u, insecure) + c, err := govmomi.NewClient(v.ctx, u, v.credentials.Insecure) if err != nil { v.Logger.Error("Unable to connect to vCenter", "error", err) return fmt.Errorf("unable to connect to vCenter : %s", err) diff --git a/log/log.go b/log/log.go index 2a17a2f..05a000f 100644 --- a/log/log.go +++ b/log/log.go @@ -65,10 +65,9 @@ func ToLevel(level string) Level { } } -// GetLevel returns the log level from the environment variable. +// GetLevel returns the default log level. func GetLevel() Level { - level := os.Getenv("LOG_LEVEL") - return ToLevel(level) + return LevelInfo } // Output represents the log output. @@ -93,8 +92,7 @@ func ToOutput(output string) Output { } } -// GetOutput returns the log output from the environment variable. +// GetOutput returns the default log output. func GetOutput() Output { - output := os.Getenv("LOG_OUTPUT") - return ToOutput(output) + return OutputText } diff --git a/main.go b/main.go index 6d82f01..a6ff273 100644 --- a/main.go +++ b/main.go @@ -2,8 +2,8 @@ package main import ( "context" + "flag" "fmt" - "log/slog" "os" "runtime" "strings" @@ -19,7 +19,6 @@ import ( "vctp/server/router" "github.com/go-co-op/gocron/v2" - "github.com/joho/godotenv" ) var ( @@ -34,22 +33,29 @@ var ( ) func main() { - // Load data from environment file - envFilename := utils.GetFilePath(".env") - err := godotenv.Load(envFilename) - if err != nil { - panic("Error loading .env file") - } + settingsPath := flag.String("settings", "/etc/dtms/vctp.yml", "Path to settings YAML") + flag.Parse() - logger := log.New( - log.GetLevel(), - log.GetOutput(), - ) + bootstrapLogger := log.New(log.LevelInfo, log.OutputText) ctx, cancel := context.WithCancel(context.Background()) + // Load settings from yaml + s := settings.New(bootstrapLogger, *settingsPath) + err := s.ReadYMLSettings() + if err != nil { + bootstrapLogger.Error("failed to open yaml settings file", "error", err, "filename", *settingsPath) + os.Exit(1) + } + + logger := log.New( + log.ToLevel(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogLevel))), + log.ToOutput(strings.ToLower(strings.TrimSpace(s.Values.Settings.LogOutput))), + ) + s.Logger = logger + // Configure database - dbDriver := os.Getenv("DB_DRIVER") + dbDriver := strings.TrimSpace(s.Values.Settings.DatabaseDriver) if dbDriver == "" { dbDriver = "sqlite" } @@ -57,12 +63,12 @@ func main() { if normalizedDriver == "" || normalizedDriver == "sqlite3" { normalizedDriver = "sqlite" } - dbURL := os.Getenv("DB_URL") + dbURL := strings.TrimSpace(s.Values.Settings.DatabaseURL) if dbURL == "" && normalizedDriver == "sqlite" { dbURL = utils.GetFilePath("db.sqlite3") } - database, err := db.New(logger, db.Config{Driver: dbDriver, DSN: dbURL}) + database, err := db.New(logger, db.Config{Driver: normalizedDriver, DSN: dbURL}) if err != nil { logger.Error("Failed to create database", "error", err) os.Exit(1) @@ -70,56 +76,36 @@ func main() { defer database.Close() //defer database.DB().Close() - if err = db.Migrate(database, dbDriver); err != nil { + if err = db.Migrate(database, normalizedDriver); err != nil { logger.Error("failed to migrate database", "error", err) os.Exit(1) } - // Load settings from yaml - settingsFile := os.Getenv("SETTINGS_FILE") - if settingsFile == "" { - settingsFile = "settings.yaml" - } - - // TODO - how to pass this to the other packages that will need this info? - s := settings.New(logger, settingsFile) - err = s.ReadYMLSettings() - //s, err := settings.ReadYMLSettings(logger, settingsFile) - if err != nil { - logger.Error("failed to open yaml settings file", "error", err, "filename", settingsFile) - //os.Exit(1) - } else { - logger.Debug("Loaded yaml settings", "contents", s) - } - // Determine bind IP - bindIP := os.Getenv("BIND_IP") + bindIP := strings.TrimSpace(s.Values.Settings.BindIP) if bindIP == "" { bindIP = utils.GetOutboundIP().String() } // Determine bind port - bindPort := os.Getenv("BIND_PORT") - if bindPort == "" { - bindPort = "9443" + bindPort := s.Values.Settings.BindPort + if bindPort == 0 { + bindPort = 9443 } bindAddress := fmt.Sprint(bindIP, ":", bindPort) //logger.Info("Will listen on address", "ip", bindIP, "port", bindPort) // Determine bind disable TLS - bindDisableTlsEnv := os.Getenv("BIND_DISABLE_TLS") - if bindDisableTlsEnv == "true" { - bindDisableTls = true - } + bindDisableTls = s.Values.Settings.BindDisableTLS // Get file names for TLS cert/key - tlsCertFilename := os.Getenv("TLS_CERT_FILE") + tlsCertFilename := strings.TrimSpace(s.Values.Settings.TLSCertFilename) if tlsCertFilename != "" { tlsCertFilename = utils.GetFilePath(tlsCertFilename) } else { tlsCertFilename = "./cert.pem" } - tlsKeyFilename := os.Getenv("TLS_KEY_FILE") + tlsKeyFilename := strings.TrimSpace(s.Values.Settings.TLSKeyFilename) if tlsKeyFilename != "" { tlsKeyFilename = utils.GetFilePath(tlsKeyFilename) } else { @@ -132,9 +118,9 @@ func main() { utils.GenerateCerts(tlsCertFilename, tlsKeyFilename) } - // Load vcenter credentials from .env + // Load vcenter credentials from serttings, decrypt if required a := secrets.New(logger, encryptionKey) - vcEp := os.Getenv("VCENTER_PASSWORD") + vcEp := strings.TrimSpace(s.Values.Settings.VcenterPassword) if len(vcEp) == 0 { logger.Error("No vcenter password configured") os.Exit(1) @@ -143,13 +129,27 @@ func main() { if err != nil { logger.Error("failed to decrypt vcenter credentials. Assuming un-encrypted", "error", err) vcPass = []byte(vcEp) + if cipherText, encErr := a.Encrypt([]byte(vcEp)); encErr != nil { + logger.Warn("failed to encrypt vcenter credentials", "error", encErr) + } else { + s.Values.Settings.VcenterPassword = cipherText + if err := s.WriteYMLSettings(); err != nil { + logger.Warn("failed to update settings with encrypted vcenter password", "error", err) + } else { + logger.Info("encrypted vcenter password stored in settings file") + } + } //os.Exit(1) } creds := vcenter.VcenterLogin{ - //insecureString := os.Getenv("VCENTER_INSECURE") - Username: os.Getenv("VCENTER_USERNAME"), + Username: strings.TrimSpace(s.Values.Settings.VcenterUsername), Password: string(vcPass), + Insecure: s.Values.Settings.VcenterInsecure, + } + if creds.Username == "" { + logger.Error("No vcenter username configured") + os.Exit(1) } // Prepare the task scheduler @@ -167,83 +167,51 @@ func main() { VcCreds: &creds, } - cronFrequencyString := os.Getenv("VCENTER_EVENT_POLLING_SECONDS") - if cronFrequencyString != "" { - cronFrequency, err = time.ParseDuration(cronFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_EVENT_POLLING_SECONDS value to time duration. Defaulting to 60s", "value", cronFrequencyString, "error", err) - cronFrequency = time.Second * 60 - } - } else { - cronFrequency = time.Second * 60 - } + cronFrequency = durationFromSeconds(s.Values.Settings.VcenterEventPollingSeconds, 60) logger.Debug("Setting VM event polling cronjob frequency to", "frequency", cronFrequency) - cronInventoryFrequencyString := os.Getenv("VCENTER_INVENTORY_POLLING_SECONDS") - if cronInventoryFrequencyString != "" { - cronInvFrequency, err = time.ParseDuration(cronInventoryFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_POLLING_SECONDS value to time duration. Defaulting to 7200", "value", cronInventoryFrequencyString, "error", err) - cronInvFrequency = time.Second * 7200 - } - } else { - cronInvFrequency = time.Second * 7200 - } + cronInvFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryPollingSeconds, 7200) logger.Debug("Setting VM inventory polling cronjob frequency to", "frequency", cronInvFrequency) - cronSnapshotFrequencyString := os.Getenv("VCENTER_INVENTORY_SNAPSHOT_SECONDS") - if cronSnapshotFrequencyString != "" { - cronSnapshotFrequency, err = time.ParseDuration(cronSnapshotFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_SNAPSHOT_SECONDS value to time duration. Defaulting to 3600", "value", cronSnapshotFrequencyString, "error", err) - cronSnapshotFrequency = time.Hour - } - } else { - cronSnapshotFrequency = time.Hour - } + cronSnapshotFrequency = durationFromSeconds(s.Values.Settings.VcenterInventorySnapshotSeconds, 3600) logger.Debug("Setting VM inventory snapshot cronjob frequency to", "frequency", cronSnapshotFrequency) - cronAggregateFrequencyString := os.Getenv("VCENTER_INVENTORY_AGGREGATE_SECONDS") - if cronAggregateFrequencyString != "" { - cronAggregateFrequency, err = time.ParseDuration(cronAggregateFrequencyString) - if err != nil { - slog.Error("Can't convert VCENTER_INVENTORY_AGGREGATE_SECONDS value to time duration. Defaulting to 86400", "value", cronAggregateFrequencyString, "error", err) - cronAggregateFrequency = time.Hour * 24 - } - } else { - cronAggregateFrequency = time.Hour * 24 - } + cronAggregateFrequency = durationFromSeconds(s.Values.Settings.VcenterInventoryAggregateSeconds, 86400) logger.Debug("Setting VM inventory aggregation cronjob frequency to", "frequency", cronAggregateFrequency) - // start background processing for events stored in events table - startsAt := time.Now().Add(time.Second * 10) - job, err := c.NewJob( - gocron.DurationJob(cronFrequency), - gocron.NewTask(func() { - ct.RunVmCheck(ctx, logger) - }), gocron.WithSingletonMode(gocron.LimitModeReschedule), - gocron.WithStartAt(gocron.WithStartDateTime(startsAt)), - ) - if err != nil { - logger.Error("failed to start event processing cron job", "error", err) - os.Exit(1) - } - logger.Debug("Created event processing cron job", "job", job.ID(), "starting_at", startsAt) + /* + // start background processing for events stored in events table + startsAt := time.Now().Add(time.Second * 10) + job, err := c.NewJob( + gocron.DurationJob(cronFrequency), + gocron.NewTask(func() { + ct.RunVmCheck(ctx, logger) + }), gocron.WithSingletonMode(gocron.LimitModeReschedule), + gocron.WithStartAt(gocron.WithStartDateTime(startsAt)), + ) + if err != nil { + logger.Error("failed to start event processing cron job", "error", err) + os.Exit(1) + } + logger.Debug("Created event processing cron job", "job", job.ID(), "starting_at", startsAt) + */ // start background checks of vcenter inventory - startsAt2 := time.Now().Add(cronInvFrequency) - job2, err := c.NewJob( - gocron.DurationJob(cronInvFrequency), - gocron.NewTask(func() { - ct.RunVcenterPoll(ctx, logger) - }), gocron.WithSingletonMode(gocron.LimitModeReschedule), - gocron.WithStartAt(gocron.WithStartDateTime(startsAt2)), - ) - if err != nil { - logger.Error("failed to start vcenter inventory cron job", "error", err) - os.Exit(1) - } - logger.Debug("Created vcenter inventory cron job", "job", job2.ID(), "starting_at", startsAt2) + /* + startsAt2 := time.Now().Add(cronInvFrequency) + job2, err := c.NewJob( + gocron.DurationJob(cronInvFrequency), + gocron.NewTask(func() { + ct.RunVcenterPoll(ctx, logger) + }), gocron.WithSingletonMode(gocron.LimitModeReschedule), + gocron.WithStartAt(gocron.WithStartDateTime(startsAt2)), + ) + if err != nil { + logger.Error("failed to start vcenter inventory cron job", "error", err) + os.Exit(1) + } + logger.Debug("Created vcenter inventory cron job", "job", job2.ID(), "starting_at", startsAt2) + */ startsAt3 := time.Now().Add(cronSnapshotFrequency) if cronSnapshotFrequency == time.Hour { @@ -292,8 +260,12 @@ func main() { } logger.Debug("Created vcenter monthly aggregation cron job", "job", job5.ID()) + snapshotCleanupCron := strings.TrimSpace(s.Values.Settings.SnapshotCleanupCron) + if snapshotCleanupCron == "" { + snapshotCleanupCron = "30 2 * * *" + } job6, err := c.NewJob( - gocron.CronJob("0 30 2 * *", false), + gocron.CronJob(snapshotCleanupCron, false), gocron.NewTask(func() { ct.RunSnapshotCleanup(ctx, logger) }), gocron.WithSingletonMode(gocron.LimitModeReschedule), @@ -325,3 +297,10 @@ func main() { os.Exit(0) } + +func durationFromSeconds(value int, fallback int) time.Duration { + if value <= 0 { + return time.Second * time.Duration(fallback) + } + return time.Second * time.Duration(value) +} diff --git a/scripts/drone.sh b/scripts/drone.sh index 78cf3f4..5837726 100755 --- a/scripts/drone.sh +++ b/scripts/drone.sh @@ -10,8 +10,13 @@ buildtime=$(date +%Y-%m-%dT%T%z) #Extract the version from yml package_version=$(grep 'version:' "$package_name.yml" | awk '{print $2}' | tr -d '"' | sed 's/^v//') -#platforms=("linux/amd64" "darwin/amd64") +host_os=$(uname -s | tr '[:upper:]' '[:lower:]') +host_arch=$(uname -m) platforms=("linux/amd64") +if [[ "$host_os" == "darwin" && ( "$host_arch" == "x86_64" || "$host_arch" == "amd64" || "$host_arch" == "arm64" ) ]]; then + platforms+=("darwin/amd64") +fi +#platforms=("linux/amd64") echo Building:: echo - Version $package_version @@ -40,6 +45,4 @@ do #sha256sum build/${output_name}.gz > build/${output_name}_checksum.txt done -nfpm package --config $package_name.yml --packager rpm --target build/ - ls -lah build diff --git a/scripts/update-swagger-ui.sh b/scripts/update-swagger-ui.sh index 62487bd..5904d25 100755 --- a/scripts/update-swagger-ui.sh +++ b/scripts/update-swagger-ui.sh @@ -4,7 +4,7 @@ set -euo pipefail # Usage: ./update-swagger-ui.sh [version] # Example: ./update-swagger-ui.sh v5.17.14 # If no version is provided, defaults below is used. -VERSION="${1:-v5.29.5}" +VERSION="${1:-v5.31.0}" TARGET_DIR="server/router/swagger-ui-dist" TARBALL_URL="https://github.com/swagger-api/swagger-ui/archive/refs/tags/${VERSION}.tar.gz" @@ -41,11 +41,19 @@ fi echo ">> Patching swagger-initializer.js to point at /swagger.json" -sed -i -E \ +if sed --version >/dev/null 2>&1; then + SED_INPLACE=(-i) +else + SED_INPLACE=(-i '') +fi + +append_validator=$'/url:[[:space:]]*"[^"]*swagger\\.json"[[:space:]]*,?$/a\\\n validatorUrl: null,' + +sed "${SED_INPLACE[@]}" -E \ -e 's#configUrl:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \ -e 's#url:[[:space:]]*["'\''"][^"'\''"]*["'\''"]#url: "/swagger.json"#' \ -e 's#urls:[[:space:]]*\[[^]]*\]#url: "/swagger.json"#' \ - -e '/url:[[:space:]]*"[^\"]*swagger\.json"[[:space:]]*,?$/a\ validatorUrl: null,' \ + -e "$append_validator" \ "$INDEX" -echo ">> Done. Files are in ${TARGET_DIR}" \ No newline at end of file +echo ">> Done. Files are in ${TARGET_DIR}" diff --git a/server/handler/snapshots.go b/server/handler/snapshots.go index a625ff0..49ef942 100644 --- a/server/handler/snapshots.go +++ b/server/handler/snapshots.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "net/url" - "strings" "vctp/components/views" "vctp/internal/report" @@ -22,7 +21,7 @@ import ( // @Failure 500 {string} string "Server error" // @Router /snapshots/hourly [get] func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_daily_", "Hourly Inventory Snapshots", views.SnapshotHourlyList) + h.renderSnapshotList(w, r, "hourly", "Hourly Inventory Snapshots", views.SnapshotHourlyList) } // SnapshotDailyList renders the daily snapshot list page. @@ -34,7 +33,7 @@ func (h *Handler) SnapshotHourlyList(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "Server error" // @Router /snapshots/daily [get] func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_daily_summary_", "Daily Inventory Snapshots", views.SnapshotDailyList) + h.renderSnapshotList(w, r, "daily", "Daily Inventory Snapshots", views.SnapshotDailyList) } // SnapshotMonthlyList renders the monthly snapshot list page. @@ -46,7 +45,7 @@ func (h *Handler) SnapshotDailyList(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "Server error" // @Router /snapshots/monthly [get] func (h *Handler) SnapshotMonthlyList(w http.ResponseWriter, r *http.Request) { - h.renderSnapshotList(w, r, "inventory_monthly_summary_", "Monthly Inventory Snapshots", views.SnapshotMonthlyList) + h.renderSnapshotList(w, r, "monthly", "Monthly Inventory Snapshots", views.SnapshotMonthlyList) } // SnapshotReportDownload streams a snapshot table as XLSX. @@ -91,28 +90,28 @@ func (h *Handler) SnapshotReportDownload(w http.ResponseWriter, r *http.Request) w.Write(reportData) } -func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, prefix string, title string, renderer func([]views.SnapshotEntry) templ.Component) { +func (h *Handler) renderSnapshotList(w http.ResponseWriter, r *http.Request, snapshotType string, title string, renderer func([]views.SnapshotEntry) templ.Component) { ctx := context.Background() - tables, err := report.ListTablesByPrefix(ctx, h.Database, prefix) + if err := report.EnsureSnapshotRegistry(ctx, h.Database); err != nil { + h.Logger.Error("Failed to ensure snapshot registry", "error", err) + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err) + return + } + records, err := report.ListSnapshots(ctx, h.Database, snapshotType) if err != nil { - h.Logger.Error("Failed to list snapshot tables", "error", err, "prefix", prefix) + h.Logger.Error("Failed to list snapshots", "error", err, "snapshot_type", snapshotType) w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "Unable to list snapshot tables: %s\n", err) return } - entries := make([]views.SnapshotEntry, 0, len(tables)) - for _, table := range tables { - if prefix == "inventory_daily_" && strings.HasPrefix(table, "inventory_daily_summary_") { - continue - } - label := table - if parsed, ok := report.FormatSnapshotLabel(prefix, table); ok { - label = parsed - } + entries := make([]views.SnapshotEntry, 0, len(records)) + for _, record := range records { + label := report.FormatSnapshotLabel(snapshotType, record.SnapshotTime, record.TableName) entries = append(entries, views.SnapshotEntry{ Label: label, - Link: "/api/report/snapshot?table=" + url.QueryEscape(table), + Link: "/api/report/snapshot?table=" + url.QueryEscape(record.TableName), }) } diff --git a/server/handler/vmImport.go b/server/handler/vmImport.go index 738b6b1..bcabefd 100644 --- a/server/handler/vmImport.go +++ b/server/handler/vmImport.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strings" "vctp/db" queries "vctp/db/queries" models "vctp/server/models" @@ -56,6 +57,17 @@ func (h *Handler) VmImport(w http.ResponseWriter, r *http.Request) { //prettyPrint(inData) } + if strings.HasPrefix(inData.Name, "vCLS-") { + h.Logger.Info("Skipping internal vCLS VM import", "vm_name", inData.Name) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(map[string]string{ + "status": "OK", + "message": fmt.Sprintf("Skipped internal VM '%s'", inData.Name), + }) + return + } + ctx := context.Background() // Query Inventory table for this VM before adding it diff --git a/server/handler/vmModifyEvent.go b/server/handler/vmModifyEvent.go index 2ff2441..c7218cd 100644 --- a/server/handler/vmModifyEvent.go +++ b/server/handler/vmModifyEvent.go @@ -441,6 +441,14 @@ func (h *Handler) AddVmToInventory(evt models.CloudEventReceived, ctx context.Co } + if strings.HasPrefix(vmObject.Name, "vCLS-") { + h.Logger.Info("Skipping internal vCLS VM", "vm_name", vmObject.Name) + if err := vc.Logout(); err != nil { + h.Logger.Error("unable to logout of vcenter", "error", err) + } + return 0, nil + } + //c.Logger.Debug("found VM") srmPlaceholder = "FALSE" // Default assumption //prettyPrint(vmObject) diff --git a/server/router/router.go b/server/router/router.go index e976547..34c3a16 100644 --- a/server/router/router.go +++ b/server/router/router.go @@ -29,6 +29,9 @@ func New(logger *slog.Logger, database db.Database, buildTime string, sha1ver st mux := http.NewServeMux() mux.Handle("/assets/", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon.ico", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon-16x16.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) + mux.Handle("/favicon-32x32.png", middleware.CacheMiddleware(http.FileServer(http.FS(dist.AssetsDir)))) mux.HandleFunc("/", h.Home) mux.HandleFunc("/api/event/vm/create", h.VmCreateEvent) mux.HandleFunc("/api/event/vm/modify", h.VmModifyEvent) diff --git a/src/vctp.default b/src/vctp.default index 633406a..9e6825c 100644 --- a/src/vctp.default +++ b/src/vctp.default @@ -1 +1 @@ -CPE_OPTS='-config /etc/dtms/vctp.yml -log-level info -log-output text' +CPE_OPTS='-settings /etc/dtms/vctp.yml' diff --git a/src/vctp.service b/src/vctp.service index eb81cd7..629ce0c 100644 --- a/src/vctp.service +++ b/src/vctp.service @@ -8,6 +8,7 @@ After=network.target Type=simple EnvironmentFile=/etc/default/vctp User=vctp +Group=dtms ExecStart=/usr/bin/vctp-linux-amd64 $CPE_OPTS ExecStartPost=/usr/bin/sleep 3 Restart=always diff --git a/src/vctp.yml b/src/vctp.yml index b6ac154..64d3f20 100644 --- a/src/vctp.yml +++ b/src/vctp.yml @@ -1,25 +1,24 @@ settings: - data_location: "/var/lib/cbs" - kickstart_location: "/var/lib/cbs/ks" - database_filename: "/var/lib/cbs/cbs.db" + log_level: "info" + log_output: "text" + database_driver: "sqlite" + database_url: "/var/lib/vctp/db.sqlite3" bind_ip: - bind_port: 443 + bind_port: 9443 bind_disable_tls: false - tls_cert_filename: "/etc/dtms/cbs.crt" - tls_key_filename: "/etc/dtms/cbs.key" - tftp_root_directory: "/var/lib/tftpboot" - tftp_images_subdirectory: "images" - replacements: - omapi: - key_name: "OMAPI" - key_secret: - special_files: - ldap_groups: - ldap_bind_address: "" - ldap_base_dn: "" - ldap_trust_cert_file: "" - ldap_disable_validation: false - ldap_insecure: false - auth_token_lifespan_hours: 2 - auth_api_key: "" - \ No newline at end of file + tls_cert_filename: "/etc/dtms/vctp.crt" + tls_key_filename: "/etc/dtms/vctp.key" + vcenter_username: "" + vcenter_password: "" + vcenter_insecure: false + vcenter_event_polling_seconds: 60 + vcenter_inventory_polling_seconds: 7200 + vcenter_inventory_snapshot_seconds: 3600 + vcenter_inventory_aggregate_seconds: 86400 + hourly_snapshot_max_age_days: 60 + daily_snapshot_max_age_months: 12 + snapshot_cleanup_cron: "30 2 * * *" + tenants_to_filter: + node_charge_clusters: + srm_activeactive_vms: + vcenter_addresses: