refactor code and improve daily cache handling of deleted VMs
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
@@ -27,6 +27,13 @@ Configuration now lives in the YAML settings file. By default the service reads
|
|||||||
vctp -settings /path/to/vctp.yml
|
vctp -settings /path/to/vctp.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
|
If you just want to run a single inventory snapshot across all configured vCenters and
|
||||||
|
exit (no scheduler/server), use:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
vctp -settings /path/to/vctp.yml -run-inventory-once
|
||||||
|
```
|
||||||
|
|
||||||
### Database Configuration
|
### Database Configuration
|
||||||
By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL
|
By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL
|
||||||
by updating the settings file:
|
by updating the settings file:
|
||||||
|
|||||||
@@ -9,12 +9,6 @@ import (
|
|||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CronTracker manages re-entry protection and status recording for cron jobs.
|
|
||||||
type CronTracker struct {
|
|
||||||
db db.Database
|
|
||||||
bindType int
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCronTracker(database db.Database) *CronTracker {
|
func NewCronTracker(database db.Database) *CronTracker {
|
||||||
return &CronTracker{
|
return &CronTracker{
|
||||||
db: database,
|
db: database,
|
||||||
|
|||||||
@@ -307,40 +307,6 @@ func (c *CronTask) aggregateDailySummaryGo(ctx context.Context, dayStart, dayEnd
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type dailyAggKey struct {
|
|
||||||
Vcenter string
|
|
||||||
VmId string
|
|
||||||
VmUuid string
|
|
||||||
Name string
|
|
||||||
}
|
|
||||||
|
|
||||||
type dailyAggVal struct {
|
|
||||||
key dailyAggKey
|
|
||||||
resourcePool string
|
|
||||||
datacenter string
|
|
||||||
cluster string
|
|
||||||
folder string
|
|
||||||
isTemplate string
|
|
||||||
poweredOn string
|
|
||||||
srmPlaceholder string
|
|
||||||
creation int64
|
|
||||||
firstSeen int64
|
|
||||||
lastSeen int64
|
|
||||||
lastDisk float64
|
|
||||||
lastVcpu int64
|
|
||||||
lastRam int64
|
|
||||||
sumVcpu int64
|
|
||||||
sumRam int64
|
|
||||||
sumDisk float64
|
|
||||||
samples int64
|
|
||||||
tinHits int64
|
|
||||||
bronzeHits int64
|
|
||||||
silverHits int64
|
|
||||||
goldHits int64
|
|
||||||
seen map[int64]struct{}
|
|
||||||
deletion int64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CronTask) scanHourlyTablesParallel(ctx context.Context, snapshots []report.SnapshotRecord) (map[dailyAggKey]*dailyAggVal, error) {
|
func (c *CronTask) scanHourlyTablesParallel(ctx context.Context, snapshots []report.SnapshotRecord) (map[dailyAggKey]*dailyAggVal, error) {
|
||||||
agg := make(map[dailyAggKey]*dailyAggVal, 1024)
|
agg := make(map[dailyAggKey]*dailyAggVal, 1024)
|
||||||
mu := sync.Mutex{}
|
mu := sync.Mutex{}
|
||||||
|
|||||||
@@ -23,31 +23,6 @@ import (
|
|||||||
"github.com/vmware/govmomi/vim25/types"
|
"github.com/vmware/govmomi/vim25/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type InventorySnapshotRow struct {
|
|
||||||
InventoryId sql.NullInt64
|
|
||||||
Name string
|
|
||||||
Vcenter string
|
|
||||||
VmId sql.NullString
|
|
||||||
EventKey sql.NullString
|
|
||||||
CloudId sql.NullString
|
|
||||||
CreationTime sql.NullInt64
|
|
||||||
DeletionTime sql.NullInt64
|
|
||||||
ResourcePool sql.NullString
|
|
||||||
Datacenter sql.NullString
|
|
||||||
Cluster sql.NullString
|
|
||||||
Folder sql.NullString
|
|
||||||
ProvisionedDisk sql.NullFloat64
|
|
||||||
VcpuCount sql.NullInt64
|
|
||||||
RamGB sql.NullInt64
|
|
||||||
IsTemplate string
|
|
||||||
PoweredOn string
|
|
||||||
SrmPlaceholder string
|
|
||||||
VmUuid sql.NullString
|
|
||||||
SnapshotTime int64
|
|
||||||
}
|
|
||||||
|
|
||||||
type snapshotTotals = db.SnapshotTotals
|
|
||||||
|
|
||||||
// RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table.
|
// RunVcenterSnapshotHourly records hourly inventory snapshots into a daily table.
|
||||||
func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) (err error) {
|
func (c *CronTask) RunVcenterSnapshotHourly(ctx context.Context, logger *slog.Logger) (err error) {
|
||||||
jobCtx := ctx
|
jobCtx := ctx
|
||||||
@@ -442,11 +417,6 @@ func filterRecordsInRange(records []report.SnapshotRecord, start, end time.Time)
|
|||||||
return filtered
|
return filtered
|
||||||
}
|
}
|
||||||
|
|
||||||
type columnDef struct {
|
|
||||||
Name string
|
|
||||||
Type string
|
|
||||||
}
|
|
||||||
|
|
||||||
var summaryUnionColumns = []string{
|
var summaryUnionColumns = []string{
|
||||||
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
`"InventoryId"`, `"Name"`, `"Vcenter"`, `"VmId"`, `"EventKey"`, `"CloudId"`, `"CreationTime"`,
|
||||||
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
`"DeletionTime"`, `"ResourcePool"`, `"Datacenter"`, `"Cluster"`, `"Folder"`,
|
||||||
|
|||||||
@@ -218,43 +218,6 @@ func (c *CronTask) aggregateMonthlySummaryGo(ctx context.Context, monthStart, mo
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type monthlyAggKey struct {
|
|
||||||
Vcenter string
|
|
||||||
VmId string
|
|
||||||
VmUuid string
|
|
||||||
Name string
|
|
||||||
}
|
|
||||||
|
|
||||||
type monthlyAggVal struct {
|
|
||||||
key monthlyAggKey
|
|
||||||
inventoryId int64
|
|
||||||
eventKey string
|
|
||||||
cloudId string
|
|
||||||
resourcePool string
|
|
||||||
datacenter string
|
|
||||||
cluster string
|
|
||||||
folder string
|
|
||||||
isTemplate string
|
|
||||||
poweredOn string
|
|
||||||
srmPlaceholder string
|
|
||||||
provisioned float64
|
|
||||||
vcpuCount int64
|
|
||||||
ramGB int64
|
|
||||||
creation int64
|
|
||||||
deletion int64
|
|
||||||
lastSnapshot time.Time
|
|
||||||
|
|
||||||
samplesPresent int64
|
|
||||||
totalSamples float64
|
|
||||||
sumVcpu float64
|
|
||||||
sumRam float64
|
|
||||||
sumDisk float64
|
|
||||||
tinWeighted float64
|
|
||||||
bronzeWeighted float64
|
|
||||||
silverWeighted float64
|
|
||||||
goldWeighted float64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CronTask) scanDailyTablesParallel(ctx context.Context, snapshots []report.SnapshotRecord) (map[monthlyAggKey]*monthlyAggVal, error) {
|
func (c *CronTask) scanDailyTablesParallel(ctx context.Context, snapshots []report.SnapshotRecord) (map[monthlyAggKey]*monthlyAggVal, error) {
|
||||||
agg := make(map[monthlyAggKey]*monthlyAggVal, 1024)
|
agg := make(map[monthlyAggKey]*monthlyAggVal, 1024)
|
||||||
mu := sync.Mutex{}
|
mu := sync.Mutex{}
|
||||||
|
|||||||
@@ -1,17 +1,3 @@
|
|||||||
package tasks
|
package tasks
|
||||||
|
|
||||||
import (
|
// Legacy placeholder: type definitions moved to types.go.
|
||||||
"log/slog"
|
|
||||||
"vctp/db"
|
|
||||||
"vctp/internal/settings"
|
|
||||||
"vctp/internal/vcenter"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CronTask stores runtime information to be used by tasks
|
|
||||||
type CronTask struct {
|
|
||||||
Logger *slog.Logger
|
|
||||||
Database db.Database
|
|
||||||
Settings *settings.Settings
|
|
||||||
VcCreds *vcenter.VcenterLogin
|
|
||||||
FirstHourlySnapshotCheck bool
|
|
||||||
}
|
|
||||||
|
|||||||
123
internal/tasks/types.go
Normal file
123
internal/tasks/types.go
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"log/slog"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"vctp/db"
|
||||||
|
"vctp/internal/settings"
|
||||||
|
"vctp/internal/vcenter"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CronTask stores runtime information to be used by tasks.
|
||||||
|
type CronTask struct {
|
||||||
|
Logger *slog.Logger
|
||||||
|
Database db.Database
|
||||||
|
Settings *settings.Settings
|
||||||
|
VcCreds *vcenter.VcenterLogin
|
||||||
|
FirstHourlySnapshotCheck bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// InventorySnapshotRow represents a single VM snapshot row.
|
||||||
|
type InventorySnapshotRow struct {
|
||||||
|
InventoryId sql.NullInt64
|
||||||
|
Name string
|
||||||
|
Vcenter string
|
||||||
|
VmId sql.NullString
|
||||||
|
EventKey sql.NullString
|
||||||
|
CloudId sql.NullString
|
||||||
|
CreationTime sql.NullInt64
|
||||||
|
DeletionTime sql.NullInt64
|
||||||
|
ResourcePool sql.NullString
|
||||||
|
Datacenter sql.NullString
|
||||||
|
Cluster sql.NullString
|
||||||
|
Folder sql.NullString
|
||||||
|
ProvisionedDisk sql.NullFloat64
|
||||||
|
VcpuCount sql.NullInt64
|
||||||
|
RamGB sql.NullInt64
|
||||||
|
IsTemplate string
|
||||||
|
PoweredOn string
|
||||||
|
SrmPlaceholder string
|
||||||
|
VmUuid sql.NullString
|
||||||
|
SnapshotTime int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshotTotals aliases DB snapshot totals for convenience.
|
||||||
|
type snapshotTotals = db.SnapshotTotals
|
||||||
|
|
||||||
|
type dailyAggKey struct {
|
||||||
|
Vcenter string
|
||||||
|
VmId string
|
||||||
|
VmUuid string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
|
||||||
|
type dailyAggVal struct {
|
||||||
|
key dailyAggKey
|
||||||
|
resourcePool string
|
||||||
|
datacenter string
|
||||||
|
cluster string
|
||||||
|
folder string
|
||||||
|
isTemplate string
|
||||||
|
poweredOn string
|
||||||
|
srmPlaceholder string
|
||||||
|
creation int64
|
||||||
|
firstSeen int64
|
||||||
|
lastSeen int64
|
||||||
|
lastDisk float64
|
||||||
|
lastVcpu int64
|
||||||
|
lastRam int64
|
||||||
|
sumVcpu int64
|
||||||
|
sumRam int64
|
||||||
|
sumDisk float64
|
||||||
|
samples int64
|
||||||
|
tinHits int64
|
||||||
|
bronzeHits int64
|
||||||
|
silverHits int64
|
||||||
|
goldHits int64
|
||||||
|
seen map[int64]struct{}
|
||||||
|
deletion int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type monthlyAggKey struct {
|
||||||
|
Vcenter string
|
||||||
|
VmId string
|
||||||
|
VmUuid string
|
||||||
|
Name string
|
||||||
|
}
|
||||||
|
|
||||||
|
type monthlyAggVal struct {
|
||||||
|
key monthlyAggKey
|
||||||
|
inventoryId int64
|
||||||
|
eventKey string
|
||||||
|
cloudId string
|
||||||
|
resourcePool string
|
||||||
|
datacenter string
|
||||||
|
cluster string
|
||||||
|
folder string
|
||||||
|
isTemplate string
|
||||||
|
poweredOn string
|
||||||
|
srmPlaceholder string
|
||||||
|
creation int64
|
||||||
|
deletion int64
|
||||||
|
lastSnapshot time.Time
|
||||||
|
provisioned float64
|
||||||
|
vcpuCount int64
|
||||||
|
ramGB int64
|
||||||
|
samplesPresent int64
|
||||||
|
totalSamples float64
|
||||||
|
sumVcpu float64
|
||||||
|
sumRam float64
|
||||||
|
sumDisk float64
|
||||||
|
tinWeighted float64
|
||||||
|
bronzeWeighted float64
|
||||||
|
silverWeighted float64
|
||||||
|
goldWeighted float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// CronTracker manages re-entry protection and status recording for cron jobs.
|
||||||
|
type CronTracker struct {
|
||||||
|
db db.Database
|
||||||
|
bindType int
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user