progress on go based aggregation

internal/tasks/inventorySnapshots.go

@@ -707,10 +707,67 @@ func snapshotFromInventory(inv queries.Inventory, snapshotTime time.Time) invent
 	}
 }
 
+func insertHourlyCache(ctx context.Context, dbConn *sqlx.DB, rows []inventorySnapshotRow) error {
+	if len(rows) == 0 {
+		return nil
+	}
+	if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil {
+		return err
+	}
+	driver := strings.ToLower(dbConn.DriverName())
+	insert := `
+INSERT INTO vm_hourly_stats (
+  "SnapshotTime","Vcenter","VmId","VmUuid","Name","CreationTime","DeletionTime","ResourcePool",
+  "Datacenter","Cluster","Folder","ProvisionedDisk","VcpuCount","RamGB","IsTemplate","PoweredOn","SrmPlaceholder"
+) VALUES (:SnapshotTime,:Vcenter,:VmId,:VmUuid,:Name,:CreationTime,:DeletionTime,:ResourcePool,
+  :Datacenter,:Cluster,:Folder,:ProvisionedDisk,:VcpuCount,:RamGB,:IsTemplate,:PoweredOn,:SrmPlaceholder)
+`
+	if driver == "sqlite" {
+		insert = strings.Replace(insert, "INSERT INTO", "INSERT OR REPLACE INTO", 1)
+	} else {
+		insert += ` ON CONFLICT ("Vcenter","VmId","SnapshotTime") DO UPDATE SET
+		  "VmUuid"=EXCLUDED."VmUuid",
+		  "Name"=EXCLUDED."Name",
+		  "CreationTime"=EXCLUDED."CreationTime",
+		  "DeletionTime"=EXCLUDED."DeletionTime",
+		  "ResourcePool"=EXCLUDED."ResourcePool",
+		  "Datacenter"=EXCLUDED."Datacenter",
+		  "Cluster"=EXCLUDED."Cluster",
+		  "Folder"=EXCLUDED."Folder",
+		  "ProvisionedDisk"=EXCLUDED."ProvisionedDisk",
+		  "VcpuCount"=EXCLUDED."VcpuCount",
+		  "RamGB"=EXCLUDED."RamGB",
+		  "IsTemplate"=EXCLUDED."IsTemplate",
+		  "PoweredOn"=EXCLUDED."PoweredOn",
+		  "SrmPlaceholder"=EXCLUDED."SrmPlaceholder"`
+	}
+	tx, err := dbConn.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	stmt, err := tx.PrepareNamedContext(ctx, insert)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+	defer stmt.Close()
+
+	for _, r := range rows {
+		if _, err := stmt.ExecContext(ctx, r); err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	return tx.Commit()
+}
+
 func insertHourlyBatch(ctx context.Context, dbConn *sqlx.DB, tableName string, rows []inventorySnapshotRow) error {
 	if len(rows) == 0 {
 		return nil
 	}
+	if err := db.EnsureVmHourlyStats(ctx, dbConn); err != nil {
+		return err
+	}
 	tx, err := dbConn.BeginTxx(ctx, nil)
 	if err != nil {
 		return err
@@ -902,6 +959,13 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
 		if err := db.UpsertVmIdentity(ctx, dbConn, url, row.VmId, row.VmUuid, row.Name, row.Cluster, startTime); err != nil {
 			c.Logger.Warn("failed to upsert vm identity", "vcenter", url, "vm_id", row.VmId, "vm_uuid", row.VmUuid, "name", row.Name, "error", err)
 		}
+		clusterName := ""
+		if row.Cluster.Valid {
+			clusterName = row.Cluster.String
+		}
+		if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, row.VmId.String, row.VmUuid.String, row.Name, clusterName, startTime); err != nil {
+			c.Logger.Warn("failed to upsert vm lifecycle cache", "vcenter", url, "vm_id", row.VmId, "vm_uuid", row.VmUuid, "name", row.Name, "error", err)
+		}
 		presentSnapshots[vm.Reference().Value] = row
 		if row.VmUuid.Valid {
 			presentByUuid[row.VmUuid.String] = struct{}{}
@@ -971,11 +1035,25 @@ func (c *CronTask) captureHourlySnapshotForVcenter(ctx context.Context, startTim
 			c.Logger.Debug("Marked VM as deleted", "name", inv.Name, "vm_id", inv.VmId.String, "vm_uuid", inv.VmUuid.String, "vcenter", url, "snapshot_time", startTime)
 			deletionsMarked = true
 		}
+		if err := db.MarkVmDeleted(ctx, dbConn, url, inv.VmId.String, inv.VmUuid.String, startTime.Unix()); err != nil {
+			c.Logger.Warn("failed to mark vm deleted in lifecycle cache", "vcenter", url, "vm_id", inv.VmId, "vm_uuid", inv.VmUuid, "error", err)
+		}
+		clusterName := ""
+		if inv.Cluster.Valid {
+			clusterName = inv.Cluster.String
+		}
+		if err := db.UpsertVmLifecycleCache(ctx, dbConn, url, inv.VmId.String, inv.VmUuid.String, inv.Name, clusterName, startTime); err != nil {
+			c.Logger.Warn("failed to upsert vm lifecycle cache (deletion path)", "vcenter", url, "vm_id", inv.VmId, "vm_uuid", inv.VmUuid, "name", inv.Name, "error", err)
+		}
 		missingCount++
 	}
 
 	c.Logger.Debug("inserting hourly snapshot batch", "vcenter", url, "rows", len(batch))
 
+	if err := insertHourlyCache(ctx, dbConn, batch); err != nil {
+		c.Logger.Warn("failed to insert hourly cache rows", "vcenter", url, "error", err)
+	}
+
 	if err := insertHourlyBatch(ctx, dbConn, tableName, batch); err != nil {
 		metrics.RecordVcenterSnapshot(url, time.Since(started), totals.VmCount, err)
 		if upErr := db.UpsertSnapshotRun(ctx, c.Database.DB(), url, startTime, false, err.Error()); upErr != nil {