nathan/vctp2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add record size to hourly snapshot page
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Nathan Coad
2026-01-14 19:43:32 +11:00
parent 9be3a3d807
commit 8df1d145f8
9 changed files with 220 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
internal/report/snapshots.go
View File

@@ -16,9 +16,10 @@ import (
)
type SnapshotRecord struct {
TableName string
SnapshotTime time.Time
SnapshotType string
TableName string
SnapshotTime time.Time
SnapshotType string
SnapshotCount int64
}
type SnapshotMigrationStats struct {
@@ -84,20 +85,36 @@ CREATE TABLE IF NOT EXISTS snapshot_registry (
id INTEGER PRIMARY KEY AUTOINCREMENT,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
)
`)
return err
if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column name") {
return err
}
return nil
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
CREATE TABLE IF NOT EXISTS snapshot_registry (
id BIGSERIAL PRIMARY KEY,
snapshot_type TEXT NOT NULL,
table_name TEXT NOT NULL UNIQUE,
snapshot_time BIGINT NOT NULL
snapshot_time BIGINT NOT NULL,
snapshot_count BIGINT NOT NULL DEFAULT 0
)
`)
return err
if err != nil {
return err
}
_, err = dbConn.ExecContext(ctx, `ALTER TABLE snapshot_registry ADD COLUMN snapshot_count BIGINT NOT NULL DEFAULT 0`)
if err != nil && !strings.Contains(strings.ToLower(err.Error()), "column \"snapshot_count\" of relation \"snapshot_registry\" already exists") {
return err
}
return nil
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
}
@@ -162,7 +179,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.HourlyRenamed++
}
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "hourly", table, snapshotTime, rowCount); err != nil {
stats.Errors++
continue
}
@@ -180,7 +198,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "daily", table, parsed); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "daily", table, parsed, rowCount); err != nil {
stats.Errors++
continue
}
@@ -198,7 +217,8 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
stats.Errors++
continue
}
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed); err != nil {
rowCount, _ := db.TableRowCount(ctx, dbConn, table)
if err := RegisterSnapshot(ctx, database, "monthly", table, parsed, rowCount); err != nil {
stats.Errors++
continue
}
@@ -211,7 +231,7 @@ func MigrateSnapshotRegistry(ctx context.Context, database db.Database) (Snapsho
return stats, nil
}
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time) error {
func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType string, tableName string, snapshotTime time.Time, snapshotCount int64) error {
if snapshotType == "" || tableName == "" {
return fmt.Errorf("snapshot type or table name is empty")
}
@@ -220,16 +240,23 @@ func RegisterSnapshot(ctx context.Context, database db.Database, snapshotType st
switch driver {
case "sqlite":
_, err := dbConn.ExecContext(ctx, `
INSERT OR IGNORE INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES (?, ?, ?)
`, snapshotType, tableName, snapshotTime.Unix())
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES (?, ?, ?, ?)
ON CONFLICT(table_name) DO UPDATE SET
snapshot_time = excluded.snapshot_time,
snapshot_type = excluded.snapshot_type,
snapshot_count = excluded.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err
case "pgx", "postgres":
_, err := dbConn.ExecContext(ctx, `
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time)
VALUES ($1, $2, $3)
ON CONFLICT (table_name) DO NOTHING
`, snapshotType, tableName, snapshotTime.Unix())
INSERT INTO snapshot_registry (snapshot_type, table_name, snapshot_time, snapshot_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT (table_name) DO UPDATE SET
snapshot_time = EXCLUDED.snapshot_time,
snapshot_type = EXCLUDED.snapshot_type,
snapshot_count = EXCLUDED.snapshot_count
`, snapshotType, tableName, snapshotTime.Unix(), snapshotCount)
return err
default:
return fmt.Errorf("unsupported driver for snapshot registry: %s", driver)
@@ -264,14 +291,14 @@ func ListSnapshots(ctx context.Context, database db.Database, snapshotType strin
switch driver {
case "sqlite":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = ?
ORDER BY snapshot_time DESC, table_name DESC
`, snapshotType)
case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = $1
ORDER BY snapshot_time DESC, table_name DESC
@@ -291,14 +318,16 @@ ORDER BY snapshot_time DESC, table_name DESC
tableName string
snapshotTime int64
recordType string
snapshotCnt int64
)
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err
}
records = append(records, SnapshotRecord{
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
SnapshotCount: snapshotCnt,
})
}
return records, rows.Err()
@@ -317,7 +346,7 @@ func ListSnapshotsByRange(ctx context.Context, database db.Database, snapshotTyp
switch driver {
case "sqlite":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = ?
AND snapshot_time >= ?
@@ -326,7 +355,7 @@ ORDER BY snapshot_time ASC, table_name ASC
`, snapshotType, startUnix, endUnix)
case "pgx", "postgres":
rows, err = dbConn.QueryxContext(ctx, `
SELECT table_name, snapshot_time, snapshot_type
SELECT table_name, snapshot_time, snapshot_type, snapshot_count
FROM snapshot_registry
WHERE snapshot_type = $1
AND snapshot_time >= $2
@@ -348,14 +377,16 @@ ORDER BY snapshot_time ASC, table_name ASC
tableName string
snapshotTime int64
recordType string
snapshotCnt int64
)
if err := rows.Scan(&tableName, &snapshotTime, &recordType); err != nil {
if err := rows.Scan(&tableName, &snapshotTime, &recordType, &snapshotCnt); err != nil {
return nil, err
}
records = append(records, SnapshotRecord{
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
TableName: tableName,
SnapshotTime: time.Unix(snapshotTime, 0),
SnapshotType: recordType,
SnapshotCount: snapshotCnt,
})
}
return records, rows.Err()