update to support postgresql and add godocs

db/db.go

@@ -3,46 +3,79 @@ package db
 import (
 	"database/sql"
 	"embed"
+	"fmt"
 	"log/slog"
 	"reflect"
-	"vctp/db/queries"
+	"strings"
 
 	"github.com/jmoiron/sqlx"
 	"github.com/pressly/goose/v3"
 )
 
-//go:embed migrations/*.sql
+//go:embed migrations migrations_postgres
 var migrations embed.FS
 
 type Database interface {
 	DB() *sqlx.DB
-	Queries() *queries.Queries
+	Queries() Querier
 	Logger() *slog.Logger
 	Close() error
 }
 
-func New(logger *slog.Logger, url string) (Database, error) {
-	db, err := newLocalDB(logger, url)
-	if err != nil {
-		return nil, err
-	}
-	if err = db.db.Ping(); err != nil {
-		return nil, err
-	}
-	return db, nil
+type Config struct {
+	Driver string
+	DSN    string
 }
 
-// Migrate runs the migrations on the database. Assumes the database is SQLite.
-func Migrate(db Database) error {
+func New(logger *slog.Logger, cfg Config) (Database, error) {
+	driver := normalizeDriver(cfg.Driver)
+	switch driver {
+	case "sqlite":
+		db, err := newLocalDB(logger, cfg.DSN)
+		if err != nil {
+			return nil, err
+		}
+		if err = db.db.Ping(); err != nil {
+			return nil, err
+		}
+		return db, nil
+	case "postgres":
+		db, err := newPostgresDB(logger, cfg.DSN)
+		if err != nil {
+			return nil, err
+		}
+		if err = db.db.Ping(); err != nil {
+			return nil, err
+		}
+		return db, nil
+	default:
+		return nil, fmt.Errorf("unsupported database driver: %s", cfg.Driver)
+	}
+}
+
+// Migrate runs the migrations on the database.
+func Migrate(db Database, driver string) error {
+	driver = normalizeDriver(driver)
 
 	goose.SetBaseFS(migrations)
 
-	if err := goose.SetDialect("sqlite3"); err != nil {
-		panic(err)
-	}
-
-	if err := goose.Up(db.DB().DB, "migrations"); err != nil {
-		panic(err)
+	switch driver {
+	case "sqlite":
+		if err := goose.SetDialect("sqlite3"); err != nil {
+			return fmt.Errorf("failed to set sqlite dialect: %w", err)
+		}
+		if err := goose.Up(db.DB().DB, "migrations"); err != nil {
+			return fmt.Errorf("failed to run sqlite migrations: %w", err)
+		}
+	case "postgres":
+		if err := goose.SetDialect("postgres"); err != nil {
+			return fmt.Errorf("failed to set postgres dialect: %w", err)
+		}
+		if err := goose.Up(db.DB().DB, "migrations_postgres"); err != nil {
+			return fmt.Errorf("failed to run postgres migrations: %w", err)
+		}
+	default:
+		return fmt.Errorf("unsupported database driver: %s", driver)
 	}
 
 	// TODO - replace with goose
@@ -69,6 +102,18 @@ func Migrate(db Database) error {
 	return nil
 }
 
+func normalizeDriver(driver string) string {
+	normalized := strings.ToLower(strings.TrimSpace(driver))
+	switch normalized {
+	case "", "sqlite3":
+		return "sqlite"
+	case "postgresql":
+		return "postgres"
+	default:
+		return normalized
+	}
+}
+
 // ConvertToSQLParams is a utility function that generically converts a struct to a corresponding sqlc-generated struct
 func ConvertToSQLParams(input interface{}, output interface{}) {
 	inputVal := reflect.ValueOf(input).Elem()

db/local.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 	"log/slog"
+	"strings"
 	"vctp/db/queries"
 
 	//_ "github.com/tursodatabase/libsql-client-go/libsql"
@@ -28,7 +29,7 @@ func (d *LocalDB) DB() *sqlx.DB {
 	return d.db
 }
 
-func (d *LocalDB) Queries() *queries.Queries {
+func (d *LocalDB) Queries() Querier {
 	return d.queries
 }
 
@@ -42,7 +43,7 @@ func (d *LocalDB) Close() error {
 	return d.db.Close()
 }
 
-func newLocalDB(logger *slog.Logger, path string) (*LocalDB, error) {
+func newLocalDB(logger *slog.Logger, dsn string) (*LocalDB, error) {
 
 	// TODO - work out if https://kerkour.com/sqlite-for-servers is possible without using sqlx
 	/*
@@ -62,8 +63,9 @@ func newLocalDB(logger *slog.Logger, path string) (*LocalDB, error) {
 		readDB.SetMaxOpenConns(max(4, runtime.NumCPU()))
 	*/
 
-	//db, err := sql.Open("libsql", "file:"+path)
-	db, err := sqlx.Open("sqlite", "file:"+path)
+	normalizedDSN := normalizeSqliteDSN(dsn)
+	//db, err := sql.Open("libsql", normalizedDSN)
+	db, err := sqlx.Open("sqlite", normalizedDSN)
 	if err != nil {
 		logger.Error("can't open database connection", "error", err)
 		return nil, err
@@ -92,3 +94,15 @@ func newLocalDB(logger *slog.Logger, path string) (*LocalDB, error) {
 
 	return &LocalDB{logger: logger, db: db, queries: queries.New(db)}, nil
 }
+
+func normalizeSqliteDSN(dsn string) string {
+	trimmed := strings.TrimSpace(dsn)
+	if trimmed == "" {
+		return "file:db.sqlite3"
+	}
+	lower := strings.ToLower(trimmed)
+	if strings.HasPrefix(lower, "file:") || strings.HasPrefix(lower, "file::memory:") || trimmed == ":memory:" {
+		return trimmed
+	}
+	return "file:" + trimmed
+}

db/migrations_postgres/20240912012927_init.sql

@@ -0,0 +1,38 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS "Inventory" (
+	"Iid" BIGSERIAL PRIMARY KEY,
+	"Name" TEXT NOT NULL,
+	"Vcenter" TEXT NOT NULL,
+	"VmId" TEXT,
+	"EventKey" TEXT,
+	"EventId" TEXT,
+	"CreationTime" BIGINT,
+	"DeletionTime" BIGINT,
+	"ResourcePool" TEXT,
+	"VmType" TEXT,
+	"Datacenter" TEXT,
+	"Cluster" TEXT,
+	"Folder" TEXT,
+	"ProvisionedDisk" REAL,
+	"InitialVcpus" INTEGER,
+	"InitialRam" INTEGER,
+	"SrmPlaceholder" INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS "Updates" (
+	"Uid" BIGSERIAL PRIMARY KEY,
+	"InventoryId" INTEGER,
+	"UpdateTime" BIGINT,
+	"UpdateType" TEXT NOT NULL,
+	"NewVcpus" INTEGER,
+	"NewRam" INTEGER,
+	"NewResourcePool" TEXT
+);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE "Inventory";
+DROP TABLE "Updates";
+-- +goose StatementEnd

db/migrations_postgres/20240912231739_extend_updates.sql

@@ -0,0 +1,11 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "EventKey" TEXT;
+ALTER TABLE "Updates" ADD COLUMN "EventId" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "EventKey";
+ALTER TABLE "Updates" DROP COLUMN "EventId";
+-- +goose StatementEnd

db/migrations_postgres/20240913021038_events.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS "Events" (
+	"Eid" BIGSERIAL PRIMARY KEY,
+	"CloudId" TEXT NOT NULL,
+	"Source" TEXT NOT NULL,
+	"EventTime" BIGINT,
+	"ChainId" TEXT NOT NULL,
+	"VmId" TEXT,
+	"EventKey" TEXT,
+	"Datacenter" TEXT,
+	"ComputeResource" TEXT,
+	"UserName" TEXT,
+	"Processed" INTEGER NOT NULL DEFAULT 0
+);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE "Events";
+-- +goose StatementEnd

db/migrations_postgres/20240913043145_extend_events.sql

@@ -0,0 +1,17 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Events" RENAME COLUMN "Datacenter" TO "DatacenterName";
+ALTER TABLE "Events" RENAME COLUMN "ComputeResource" TO "ComputeResourceName";
+ALTER TABLE "Events" ADD COLUMN "DatacenterId" TEXT;
+ALTER TABLE "Events" ADD COLUMN "ComputeResourceId" TEXT;
+ALTER TABLE "Events" ADD COLUMN "VmName" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Events" DROP COLUMN "VmName";
+ALTER TABLE "Events" DROP COLUMN "ComputeResourceId";
+ALTER TABLE "Events" DROP COLUMN "DatacenterId";
+ALTER TABLE "Events" RENAME COLUMN "ComputeResourceName" TO "ComputeResource";
+ALTER TABLE "Events" RENAME COLUMN "DatacenterName" TO "Datacenter";
+-- +goose StatementEnd

db/migrations_postgres/20240915015710_events_type.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Events" ADD COLUMN "EventType" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "EventType";
+-- +goose StatementEnd

db/migrations_postgres/20240915232747_extend_inventory.sql

@@ -0,0 +1,11 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Inventory" ADD COLUMN "IsTemplate" INTEGER;
+ALTER TABLE "Inventory" ADD COLUMN "PowerState" INTEGER;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Inventory" DROP COLUMN "PowerState";
+ALTER TABLE "Inventory" DROP COLUMN "IsTemplate";
+-- +goose StatementEnd

db/migrations_postgres/20240916041639_rename_eventid.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Inventory" RENAME COLUMN "EventId" TO "CloudId";
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Inventory" RENAME COLUMN "CloudId" TO "EventId";
+-- +goose StatementEnd

db/migrations_postgres/20240916045259_updates_disk.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "NewProvisionedDisk" REAL;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "NewProvisionedDisk";
+-- +goose StatementEnd

db/migrations_postgres/20240926022714_updates_user.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "UserName" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "UserName";
+-- +goose StatementEnd

db/migrations_postgres/20240927002029_change_inventory.sql

@@ -0,0 +1,55 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Inventory" RENAME COLUMN "IsTemplate" TO "IsTemplate_old";
+ALTER TABLE "Inventory" RENAME COLUMN "PowerState" TO "PowerState_old";
+ALTER TABLE "Inventory" RENAME COLUMN "SrmPlaceholder" TO "SrmPlaceholder_old";
+ALTER TABLE "Inventory" ADD COLUMN "IsTemplate" TEXT NOT NULL DEFAULT 'FALSE';
+ALTER TABLE "Inventory" ADD COLUMN "PoweredOn" TEXT NOT NULL DEFAULT 'FALSE';
+ALTER TABLE "Inventory" ADD COLUMN "SrmPlaceholder" TEXT NOT NULL DEFAULT 'FALSE';
+UPDATE "Inventory"
+SET "IsTemplate" = CASE
+	WHEN "IsTemplate_old" = 1 THEN 'TRUE'
+	ELSE 'FALSE'
+END;
+UPDATE "Inventory"
+SET "PoweredOn" = CASE
+	WHEN "PowerState_old" = 1 THEN 'TRUE'
+	ELSE 'FALSE'
+END;
+UPDATE "Inventory"
+SET "SrmPlaceholder" = CASE
+	WHEN "SrmPlaceholder_old" = 1 THEN 'TRUE'
+	ELSE 'FALSE'
+END;
+ALTER TABLE "Inventory" DROP COLUMN "IsTemplate_old";
+ALTER TABLE "Inventory" DROP COLUMN "PowerState_old";
+ALTER TABLE "Inventory" DROP COLUMN "SrmPlaceholder_old";
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Inventory" RENAME COLUMN "IsTemplate" TO "IsTemplate_old";
+ALTER TABLE "Inventory" RENAME COLUMN "PoweredOn" TO "PoweredOn_old";
+ALTER TABLE "Inventory" RENAME COLUMN "SrmPlaceholder" TO "SrmPlaceholder_old";
+ALTER TABLE "Inventory" ADD COLUMN "IsTemplate" INTEGER;
+ALTER TABLE "Inventory" ADD COLUMN "PowerState" INTEGER;
+ALTER TABLE "Inventory" ADD COLUMN "SrmPlaceholder" INTEGER;
+UPDATE "Inventory"
+SET "IsTemplate" = CASE
+	WHEN "IsTemplate_old" = 'TRUE' THEN 1
+	ELSE 0
+END;
+UPDATE "Inventory"
+SET "PowerState" = CASE
+	WHEN "PoweredOn_old" = 'TRUE' THEN 1
+	ELSE 0
+END;
+UPDATE "Inventory"
+SET "SrmPlaceholder" = CASE
+	WHEN "SrmPlaceholder_old" = 'TRUE' THEN 1
+	ELSE 0
+END;
+ALTER TABLE "Inventory" DROP COLUMN "IsTemplate_old";
+ALTER TABLE "Inventory" DROP COLUMN "PoweredOn_old";
+ALTER TABLE "Inventory" DROP COLUMN "SrmPlaceholder_old";
+-- +goose StatementEnd

db/migrations_postgres/20240930012450_add_uuid.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Inventory" ADD COLUMN "VmUuid" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Inventory" DROP COLUMN "VmUuid";
+-- +goose StatementEnd

db/migrations_postgres/20240930031506_add_table.sql

@@ -0,0 +1,18 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TABLE IF NOT EXISTS "InventoryHistory" (
+	"Hid" BIGSERIAL PRIMARY KEY,
+	"InventoryId" INTEGER,
+	"ReportDate" BIGINT,
+	"UpdateTime" BIGINT,
+	"PreviousVcpus" INTEGER,
+	"PreviousRam" INTEGER,
+	"PreviousResourcePool" TEXT,
+	"PreviousProvisionedDisk" REAL
+);
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+DROP TABLE "InventoryHistory";
+-- +goose StatementEnd

db/migrations_postgres/20241001222729_add_placeholder.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "PlaceholderChange" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "PlaceholderChange";
+-- +goose StatementEnd

db/migrations_postgres/20241002032439_add_name.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "Name" TEXT;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "Name";
+-- +goose StatementEnd

db/migrations_postgres/20241002033323_add_change.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE "Updates" ADD COLUMN "RawChangeString" BYTEA;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+ALTER TABLE "Updates" DROP COLUMN "RawChangeString";
+-- +goose StatementEnd

db/postgres.go

@@ -0,0 +1,79 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"regexp"
+	"strings"
+	"vctp/db/queries"
+
+	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/jmoiron/sqlx"
+)
+
+type PostgresDB struct {
+	logger  *slog.Logger
+	db      *sqlx.DB
+	queries *queries.Queries
+}
+
+var _ Database = (*PostgresDB)(nil)
+
+func (d *PostgresDB) DB() *sqlx.DB {
+	return d.db
+}
+
+func (d *PostgresDB) Queries() Querier {
+	return d.queries
+}
+
+func (d *PostgresDB) Logger() *slog.Logger {
+	return d.logger
+}
+
+func (d *PostgresDB) Close() error {
+	return d.db.Close()
+}
+
+func newPostgresDB(logger *slog.Logger, dsn string) (*PostgresDB, error) {
+	if strings.TrimSpace(dsn) == "" {
+		return nil, fmt.Errorf("postgres DSN is required")
+	}
+	db, err := sqlx.Open("pgx", dsn)
+	if err != nil {
+		return nil, err
+	}
+	db.SetMaxOpenConns(10)
+
+	rebindDB := rebindDBTX{db: db}
+	return &PostgresDB{logger: logger, db: db, queries: queries.New(rebindDB)}, nil
+}
+
+type rebindDBTX struct {
+	db *sqlx.DB
+}
+
+func (r rebindDBTX) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
+	return r.db.ExecContext(ctx, rebindQuery(query), args...)
+}
+
+func (r rebindDBTX) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) {
+	return r.db.PrepareContext(ctx, rebindQuery(query))
+}
+
+func (r rebindDBTX) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) {
+	return r.db.QueryContext(ctx, rebindQuery(query), args...)
+}
+
+func (r rebindDBTX) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
+	return r.db.QueryRowContext(ctx, rebindQuery(query), args...)
+}
+
+var numberedPlaceholderRe = regexp.MustCompile(`\?\d+`)
+
+func rebindQuery(query string) string {
+	unindexed := numberedPlaceholderRe.ReplaceAllString(query, "?")
+	return sqlx.Rebind(sqlx.DOLLAR, unindexed)
+}

db/querier.go

@@ -0,0 +1,35 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"vctp/db/queries"
+)
+
+// Querier abstracts sqlc-generated queries so multiple database backends can share call sites.
+type Querier interface {
+	CleanupUpdates(ctx context.Context, arg queries.CleanupUpdatesParams) error
+	CleanupUpdatesNullVm(ctx context.Context) error
+	CreateEvent(ctx context.Context, arg queries.CreateEventParams) (queries.Events, error)
+	CreateInventory(ctx context.Context, arg queries.CreateInventoryParams) (queries.Inventory, error)
+	CreateInventoryHistory(ctx context.Context, arg queries.CreateInventoryHistoryParams) (queries.InventoryHistory, error)
+	CreateUpdate(ctx context.Context, arg queries.CreateUpdateParams) (queries.Updates, error)
+	GetInventoryByName(ctx context.Context, name string) ([]queries.Inventory, error)
+	GetInventoryByVcenter(ctx context.Context, vcenter string) ([]queries.Inventory, error)
+	GetInventoryEventId(ctx context.Context, cloudid sql.NullString) (queries.Inventory, error)
+	GetInventoryVcUrl(ctx context.Context, vc string) ([]queries.Inventory, error)
+	GetInventoryVmId(ctx context.Context, arg queries.GetInventoryVmIdParams) (queries.Inventory, error)
+	GetInventoryVmUuid(ctx context.Context, arg queries.GetInventoryVmUuidParams) (queries.Inventory, error)
+	GetReportInventory(ctx context.Context) ([]queries.Inventory, error)
+	GetReportUpdates(ctx context.Context) ([]queries.Updates, error)
+	GetVmUpdates(ctx context.Context, arg queries.GetVmUpdatesParams) ([]queries.Updates, error)
+	InventoryCleanup(ctx context.Context, arg queries.InventoryCleanupParams) error
+	InventoryCleanupTemplates(ctx context.Context) error
+	InventoryCleanupVcenter(ctx context.Context, vc string) error
+	InventoryMarkDeleted(ctx context.Context, arg queries.InventoryMarkDeletedParams) error
+	InventoryUpdate(ctx context.Context, arg queries.InventoryUpdateParams) error
+	ListEvents(ctx context.Context) ([]queries.Events, error)
+	ListInventory(ctx context.Context) ([]queries.Inventory, error)
+	ListUnprocessedEvents(ctx context.Context, eventtime sql.NullInt64) ([]queries.Events, error)
+	UpdateEventsProcessed(ctx context.Context, eid int64) error
+}

db/queries/query.sql

@@ -60,7 +60,7 @@ RETURNING *;
 
 -- name: InventoryCleanupTemplates :exec
 DELETE FROM "Inventory"
-WHERE "IsTemplate" = "TRUE"
+WHERE "IsTemplate" = 'TRUE'
 RETURNING *;
 
 -- name: CreateUpdate :one
@@ -118,4 +118,4 @@ INSERT INTO "InventoryHistory" (
 ) VALUES(
     ?, ?, ?, ?, ?, ?, ?
 )
-RETURNING *;
+RETURNING *;

db/queries/query.sql.go

@@ -694,7 +694,7 @@ func (q *Queries) InventoryCleanup(ctx context.Context, arg InventoryCleanupPara
 
 const inventoryCleanupTemplates = `-- name: InventoryCleanupTemplates :exec
 DELETE FROM "Inventory"
-WHERE "IsTemplate" = "TRUE"
+WHERE "IsTemplate" = 'TRUE'
 RETURNING Iid, Name, Vcenter, VmId, EventKey, CloudId, CreationTime, DeletionTime, ResourcePool, VmType, Datacenter, Cluster, Folder, ProvisionedDisk, InitialVcpus, InitialRam, IsTemplate, PoweredOn, SrmPlaceholder, VmUuid
 `