From 5dcc11e5e041f9310edf00ace89d7bdf89eb4095 Mon Sep 17 00:00:00 2001
From: Nathan Coad <nathan@thecoads.com>
Date: Fri, 6 Feb 2026 08:53:36 +1100
Subject: [PATCH] reduce unnecessary sqlite indexes

---
 README.md     |  7 ++++
 db/helpers.go | 88 +++++++++++++++++++++++++++++++++++++++++++++++++--
 main.go       | 10 ++++++
 3 files changed, 103 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c6ad12d..f36ca56 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,13 @@ exit (no scheduler/server), use:
 vctp -settings /path/to/vctp.yml -run-inventory
 ```
 
+If you want a one-time SQLite cleanup to drop low-value hourly snapshot indexes and exit,
+use:
+
+```shell
+vctp -settings /path/to/vctp.yml -db-cleanup
+```
+
 ## Database Configuration
 By default the app uses SQLite and creates/opens `db.sqlite3`. You can opt into PostgreSQL
 by updating the settings file:
diff --git a/db/helpers.go b/db/helpers.go
index b70264b..0d9c145 100644
--- a/db/helpers.go
+++ b/db/helpers.go
@@ -354,8 +354,12 @@ func EnsureSnapshotIndexes(ctx context.Context, dbConn *sqlx.DB, tableName strin
 	driver := strings.ToLower(dbConn.DriverName())
 	indexes := []string{
 		fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName),
-		fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_snapshottime_idx ON %s ("SnapshotTime")`, tableName, tableName),
-		fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName),
+	}
+	if driver != "sqlite" {
+		indexes = append(indexes,
+			fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_snapshottime_idx ON %s ("SnapshotTime")`, tableName, tableName),
+			fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_resourcepool_idx ON %s ("ResourcePool")`, tableName, tableName),
+		)
 	}
 	// PG-specific helpful indexes; safe no-ops on SQLite if executed, but keep them gated to reduce file bloat.
 	if driver == "pgx" || driver == "postgres" {
@@ -373,6 +377,86 @@ func EnsureSnapshotIndexes(ctx context.Context, dbConn *sqlx.DB, tableName strin
 	return nil
 }
 
+func listTablesByPrefix(ctx context.Context, dbConn *sqlx.DB, prefix string) ([]string, error) {
+	driver := strings.ToLower(dbConn.DriverName())
+	pattern := prefix + "%"
+	switch driver {
+	case "sqlite":
+		rows, err := dbConn.QueryxContext(ctx, `
+SELECT name
+FROM sqlite_master
+WHERE type = 'table'
+  AND name LIKE ?
+ORDER BY name DESC
+`, pattern)
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+		var tables []string
+		for rows.Next() {
+			var name string
+			if err := rows.Scan(&name); err != nil {
+				return nil, err
+			}
+			tables = append(tables, name)
+		}
+		return tables, rows.Err()
+	case "pgx", "postgres":
+		rows, err := dbConn.QueryxContext(ctx, `
+SELECT tablename
+FROM pg_catalog.pg_tables
+WHERE schemaname = 'public'
+  AND tablename LIKE $1
+ORDER BY tablename DESC
+`, pattern)
+		if err != nil {
+			return nil, err
+		}
+		defer rows.Close()
+		var tables []string
+		for rows.Next() {
+			var name string
+			if err := rows.Scan(&name); err != nil {
+				return nil, err
+			}
+			tables = append(tables, name)
+		}
+		return tables, rows.Err()
+	default:
+		return nil, fmt.Errorf("unsupported driver: %s", driver)
+	}
+}
+
+// CleanupHourlySnapshotIndexes drops low-value per-table indexes on hourly snapshot tables.
+func CleanupHourlySnapshotIndexes(ctx context.Context, dbConn *sqlx.DB) (int, error) {
+	driver := strings.ToLower(dbConn.DriverName())
+	if driver != "sqlite" {
+		return 0, fmt.Errorf("hourly snapshot index cleanup is only supported for sqlite")
+	}
+	tables, err := listTablesByPrefix(ctx, dbConn, "inventory_hourly_")
+	if err != nil {
+		return 0, err
+	}
+	dropped := 0
+	for _, tableName := range tables {
+		if _, err := SafeTableName(tableName); err != nil {
+			continue
+		}
+		indexes := []string{
+			fmt.Sprintf(`DROP INDEX IF EXISTS %s_snapshottime_idx`, tableName),
+			fmt.Sprintf(`DROP INDEX IF EXISTS %s_resourcepool_idx`, tableName),
+		}
+		for _, stmt := range indexes {
+			if _, err := execLog(ctx, dbConn, stmt); err != nil {
+				return dropped, err
+			}
+			dropped++
+		}
+	}
+	return dropped, nil
+}
+
 // BackfillSerialColumn sets missing values in a serial-like column for Postgres tables.
 func BackfillSerialColumn(ctx context.Context, dbConn *sqlx.DB, tableName, columnName string) error {
 	if err := ValidateTableName(tableName); err != nil {
diff --git a/main.go b/main.go
index 1a5abf1..300b89e 100644
--- a/main.go
+++ b/main.go
@@ -39,6 +39,7 @@ const fallbackEncryptionKey = "5L1l3B5KvwOCzUHMAlCgsgUTRAYMfSpa"
 func main() {
 	settingsPath := flag.String("settings", "/etc/dtms/vctp.yml", "Path to settings YAML")
 	runInventory := flag.Bool("run-inventory", false, "Run a single inventory snapshot across all configured vCenters and exit")
+	dbCleanup := flag.Bool("db-cleanup", false, "Run a one-time cleanup to drop low-value hourly snapshot indexes and exit")
 	flag.Parse()
 
 	bootstrapLogger := log.New(log.LevelInfo, log.OutputText)
@@ -87,6 +88,15 @@ func main() {
 		logger.Error("failed to migrate database", "error", err)
 		os.Exit(1)
 	}
+	if *dbCleanup {
+		dropped, err := db.CleanupHourlySnapshotIndexes(ctx, database.DB())
+		if err != nil {
+			logger.Error("failed to cleanup hourly snapshot indexes", "error", err)
+			os.Exit(1)
+		}
+		logger.Info("completed hourly snapshot index cleanup", "indexes_dropped", dropped)
+		return
+	}
 
 	// Determine bind IP
 	bindIP := strings.TrimSpace(s.Values.Settings.BindIP)