more optimisation

db/helpers.go

@@ -277,6 +277,12 @@ func EnsureSnapshotTable(ctx context.Context, dbConn *sqlx.DB, tableName string)
 	}
 
 	_, err := dbConn.ExecContext(ctx, ddl)
+	if err != nil {
+		return err
+	}
+
+	index := fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_vm_vcenter_idx ON %s ("VmId","Vcenter")`, tableName, tableName)
+	_, err = dbConn.ExecContext(ctx, index)
 	return err
 }
 
@@ -303,6 +309,67 @@ func BackfillSerialColumn(ctx context.Context, dbConn *sqlx.DB, tableName, colum
 	return nil
 }
 
+// ApplySQLiteTuning applies lightweight WAL/synchronous tweaks for better concurrency in non-prod contexts.
+func ApplySQLiteTuning(ctx context.Context, dbConn *sqlx.DB) {
+	if strings.ToLower(dbConn.DriverName()) != "sqlite" {
+		return
+	}
+	// Best-effort pragmas; ignore errors to stay safe in constrained environments.
+	pragmas := []string{
+		`PRAGMA journal_mode=WAL;`,
+		`PRAGMA synchronous=NORMAL;`,
+		`PRAGMA temp_store=MEMORY;`,
+	}
+	for _, pragma := range pragmas {
+		_, _ = dbConn.ExecContext(ctx, pragma)
+	}
+}
+
+// CheckMigrationState ensures goose migrations are present and not dirty.
+func CheckMigrationState(ctx context.Context, dbConn *sqlx.DB) error {
+	driver := strings.ToLower(dbConn.DriverName())
+	var tableExists bool
+	switch driver {
+	case "sqlite":
+		err := dbConn.GetContext(ctx, &tableExists, `
+SELECT COUNT(1) > 0 FROM sqlite_master WHERE type='table' AND name='goose_db_version'
+`)
+		if err != nil {
+			return err
+		}
+	case "pgx", "postgres":
+		err := dbConn.GetContext(ctx, &tableExists, `
+SELECT EXISTS (
+  SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = 'goose_db_version'
+)
+`)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unsupported driver for migration check: %s", driver)
+	}
+
+	if !tableExists {
+		return fmt.Errorf("goose_db_version table not found; database migrations may not be applied")
+	}
+
+	var dirty bool
+	err := dbConn.GetContext(ctx, &dirty, `
+SELECT NOT is_applied
+FROM goose_db_version
+ORDER BY id DESC
+LIMIT 1
+`)
+	if err != nil {
+		return err
+	}
+	if dirty {
+		return fmt.Errorf("database migrations are in a dirty state; please resolve goose_db_version")
+	}
+	return nil
+}
+
 // BuildDailySummaryInsert returns the SQL to aggregate hourly snapshots into a daily summary table.
 func BuildDailySummaryInsert(tableName string, unionQuery string) (string, error) {
 	if _, err := SafeTableName(tableName); err != nil {

db/migrations/20250116102000_snapshot_registry_indexes.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE INDEX IF NOT EXISTS idx_snapshot_registry_type_time ON snapshot_registry (snapshot_type, snapshot_time);
+
+-- +goose Down
+DROP INDEX IF EXISTS idx_snapshot_registry_type_time;

db/migrations_postgres/20250116102000_snapshot_registry_indexes.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+CREATE INDEX IF NOT EXISTS idx_snapshot_registry_type_time ON snapshot_registry (snapshot_type, snapshot_time);
+
+-- +goose Down
+DROP INDEX IF EXISTS idx_snapshot_registry_type_time;

db/queries/models.go

@@ -69,10 +69,11 @@ type PragmaTableInfo struct {
 }
 
 type SnapshotRegistry struct {
-	ID           int64  `db:"id" json:"id"`
-	SnapshotType string `db:"snapshot_type" json:"snapshot_type"`
-	TableName    string `db:"table_name" json:"table_name"`
-	SnapshotTime int64  `db:"snapshot_time" json:"snapshot_time"`
+	ID            int64  `db:"id" json:"id"`
+	SnapshotType  string `db:"snapshot_type" json:"snapshot_type"`
+	TableName     string `db:"table_name" json:"table_name"`
+	SnapshotTime  int64  `db:"snapshot_time" json:"snapshot_time"`
+	SnapshotCount int64  `db:"snapshot_count" json:"snapshot_count"`
 }
 
 type SqliteMaster struct {

db/schema.sql

@@ -73,6 +73,7 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
 	"snapshot_time" INTEGER NOT NULL,
 	"snapshot_count" BIGINT NOT NULL DEFAULT 0
 );
+CREATE INDEX IF NOT EXISTS idx_snapshot_registry_type_time ON snapshot_registry (snapshot_type, snapshot_time);
 
 -- The following tables are declared for sqlc type-checking only.
 -- Do not apply this file as a migration.