update for 4 hour rain forecast

This commit is contained in:
2026-04-06 18:32:33 +10:00
parent fb50c8ed71
commit 3a7309b2cf
20 changed files with 716 additions and 132 deletions
+9 -4
View File
@@ -76,7 +76,8 @@ TimescaleDB schema is initialized from `db/init/001_schema.sql` and includes:
- `observations_ws90` (hypertable): raw WS90 observations with payload metadata, plus the full JSON payload (`payload_json`). - `observations_ws90` (hypertable): raw WS90 observations with payload metadata, plus the full JSON payload (`payload_json`).
- `observations_baro` (hypertable): barometric pressure observations from other MQTT topics. - `observations_baro` (hypertable): barometric pressure observations from other MQTT topics.
- `forecast_openmeteo_hourly` (hypertable): hourly forecast points keyed by `(site, model, retrieved_at, ts)`. - `forecast_openmeteo_hourly` (hypertable): hourly forecast points keyed by `(site, model, retrieved_at, ts)`.
- `predictions_rain_1h` (hypertable): model probability + decision + realized outcome fields. - `predictions_rain_1h` (hypertable): legacy 1-hour model probability + decision + realized outcome fields.
- `predictions_rain_4h` (hypertable): 4-hour model probability + decision + realized outcome fields.
- Continuous aggregates: - Continuous aggregates:
- `cagg_ws90_1m`: 1minute rollups (avg/min/max for temp, humidity, wind, uvi, light, rain). - `cagg_ws90_1m`: 1minute rollups (avg/min/max for temp, humidity, wind, uvi, light, rain).
- `cagg_ws90_5m`: 5minute rollups (same metrics as `cagg_ws90_1m`). - `cagg_ws90_5m`: 5minute rollups (same metrics as `cagg_ws90_1m`).
@@ -86,14 +87,18 @@ Retention/compression:
- `observations_baro` has a 90day retention policy and compression after 7 days. - `observations_baro` has a 90day retention policy and compression after 7 days.
### Existing databases ### Existing databases
If youre on an existing database, youll need to apply the new table definition once (the init SQL only runs on a fresh DB). Example: If youre on an existing database, apply the 4-hour prediction table migration:
```sh
docker compose exec -T timescaledb psql -U postgres -d micrometeo -f /docker-entrypoint-initdb.d/003_rain_predictions_4h.sql
```
If you also need to backfill older schema objects, you can still re-run:
```sh ```sh
docker compose exec -T timescaledb psql -U postgres -d micrometeo -f /docker-entrypoint-initdb.d/001_schema.sql docker compose exec -T timescaledb psql -U postgres -d micrometeo -f /docker-entrypoint-initdb.d/001_schema.sql
``` ```
Or copy just the `observations_baro` section into a manual `psql -c`.
For rain-model monitoring views, also apply `db/init/002_rain_monitoring_views.sql` on existing DBs: For rain-model monitoring views, also apply `db/init/002_rain_monitoring_views.sql` on existing DBs:
```sh ```sh
+16 -3
View File
@@ -179,16 +179,29 @@ func (s *webServer) handleDashboard(w http.ResponseWriter, r *http.Request) {
return return
} }
const rainModelName = "rain_next_1h" const rainModelName = "rain_next_4h"
const rainModelHorizonHours = 4
latestRainPrediction, err := s.db.LatestRainPrediction(r.Context(), s.site.Name, rainModelName) latestRainPrediction, err := s.db.LatestRainPrediction(
r.Context(),
s.site.Name,
rainModelName,
rainModelHorizonHours,
)
if err != nil { if err != nil {
http.Error(w, "failed to query latest rain prediction", http.StatusInternalServerError) http.Error(w, "failed to query latest rain prediction", http.StatusInternalServerError)
log.Printf("web dashboard latest rain prediction error: %v", err) log.Printf("web dashboard latest rain prediction error: %v", err)
return return
} }
rainPredictionRange, err := s.db.RainPredictionSeriesRange(r.Context(), s.site.Name, rainModelName, start, end) rainPredictionRange, err := s.db.RainPredictionSeriesRange(
r.Context(),
s.site.Name,
rainModelName,
rainModelHorizonHours,
start,
end,
)
if err != nil { if err != nil {
http.Error(w, "failed to query rain predictions", http.StatusInternalServerError) http.Error(w, "failed to query rain predictions", http.StatusInternalServerError)
log.Printf("web dashboard rain prediction range error: %v", err) log.Printf("web dashboard rain prediction range error: %v", err)
+1 -1
View File
@@ -979,7 +979,7 @@ function renderDashboard(data) {
data: { data: {
datasets: [ datasets: [
{ {
label: rainPredictions.length ? "model rain probability (%)" : "heuristic rain probability (%)", label: rainPredictions.length ? "model rain probability next 4h (%)" : "heuristic rain probability (%)",
data: rainPredictions.length ? buildRainProbabilitySeriesFromPredictions(rainPredictions) : buildRainProbabilitySeries(obsFiltered), data: rainPredictions.length ? buildRainProbabilitySeriesFromPredictions(rainPredictions) : buildRainProbabilitySeries(obsFiltered),
borderColor: colors.rain, borderColor: colors.rain,
backgroundColor: "rgba(78, 168, 222, 0.18)", backgroundColor: "rgba(78, 168, 222, 0.18)",
+1 -1
View File
@@ -70,7 +70,7 @@
<div class="value" id="live-pressure">--</div> <div class="value" id="live-pressure">--</div>
</div> </div>
<div class="metric"> <div class="metric">
<div class="label">Rain 1h %</div> <div class="label">Rain 4h %</div>
<div class="value" id="live-rain-prob">--</div> <div class="value" id="live-rain-prob">--</div>
</div> </div>
<div class="metric"> <div class="metric">
+28
View File
@@ -112,6 +112,34 @@ CREATE INDEX IF NOT EXISTS idx_predictions_rain_1h_site_ts
CREATE INDEX IF NOT EXISTS idx_predictions_rain_1h_pending_eval CREATE INDEX IF NOT EXISTS idx_predictions_rain_1h_pending_eval
ON predictions_rain_1h(site, evaluated_at, ts DESC); ON predictions_rain_1h(site, evaluated_at, ts DESC);
-- Rain model predictions (next 4h)
CREATE TABLE IF NOT EXISTS predictions_rain_4h (
ts TIMESTAMPTZ NOT NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
site TEXT NOT NULL,
model_name TEXT NOT NULL,
model_version TEXT NOT NULL,
threshold DOUBLE PRECISION NOT NULL,
probability DOUBLE PRECISION NOT NULL,
predict_rain BOOLEAN NOT NULL,
rain_next_4h_mm_actual DOUBLE PRECISION,
rain_next_4h_actual BOOLEAN,
evaluated_at TIMESTAMPTZ,
metadata JSONB,
PRIMARY KEY (site, model_name, model_version, ts)
);
SELECT create_hypertable('predictions_rain_4h', 'ts', if_not_exists => TRUE);
CREATE INDEX IF NOT EXISTS idx_predictions_rain_4h_site_ts
ON predictions_rain_4h(site, ts DESC);
CREATE INDEX IF NOT EXISTS idx_predictions_rain_4h_pending_eval
ON predictions_rain_4h(site, evaluated_at, ts DESC);
-- Raw retention: 90 days -- Raw retention: 90 days
DO $$ DO $$
BEGIN BEGIN
+79 -15
View File
@@ -100,44 +100,104 @@ FROM baseline;
CREATE OR REPLACE VIEW rain_prediction_drift_daily AS CREATE OR REPLACE VIEW rain_prediction_drift_daily AS
WITH all_predictions AS (
SELECT
1::INT AS horizon_hours,
ts,
site,
model_name,
model_version,
threshold,
probability,
predict_rain
FROM predictions_rain_1h
UNION ALL
SELECT
4::INT AS horizon_hours,
ts,
site,
model_name,
model_version,
threshold,
probability,
predict_rain
FROM predictions_rain_4h
)
SELECT SELECT
time_bucket(INTERVAL '1 day', ts) AS day, time_bucket(INTERVAL '1 day', ts) AS day,
site, site,
model_name, model_name,
model_version, model_version,
horizon_hours,
count(*) AS prediction_rows, count(*) AS prediction_rows,
avg(probability) AS probability_mean, avg(probability) AS probability_mean,
stddev_samp(probability) AS probability_stddev, stddev_samp(probability) AS probability_stddev,
avg(CASE WHEN predict_rain THEN 1.0 ELSE 0.0 END) AS predicted_positive_rate, avg(CASE WHEN predict_rain THEN 1.0 ELSE 0.0 END) AS predicted_positive_rate,
avg(threshold) AS threshold_mean avg(threshold) AS threshold_mean
FROM predictions_rain_1h FROM all_predictions
GROUP BY 1,2,3,4; GROUP BY 1,2,3,4,5;
CREATE OR REPLACE VIEW rain_calibration_drift_daily AS CREATE OR REPLACE VIEW rain_calibration_drift_daily AS
WITH all_predictions AS (
SELECT
1::INT AS horizon_hours,
ts,
site,
model_name,
model_version,
probability,
predict_rain,
rain_next_1h_actual AS rain_actual
FROM predictions_rain_1h
UNION ALL
SELECT
4::INT AS horizon_hours,
ts,
site,
model_name,
model_version,
probability,
predict_rain,
rain_next_4h_actual AS rain_actual
FROM predictions_rain_4h
)
SELECT SELECT
time_bucket(INTERVAL '1 day', ts) AS day, time_bucket(INTERVAL '1 day', ts) AS day,
site, site,
model_name, model_name,
model_version, model_version,
count(*) FILTER (WHERE rain_next_1h_actual IS NOT NULL) AS evaluated_rows, horizon_hours,
avg(probability) FILTER (WHERE rain_next_1h_actual IS NOT NULL) AS mean_probability, count(*) FILTER (WHERE rain_actual IS NOT NULL) AS evaluated_rows,
avg(CASE WHEN rain_next_1h_actual THEN 1.0 ELSE 0.0 END) FILTER (WHERE rain_next_1h_actual IS NOT NULL) AS observed_positive_rate, avg(probability) FILTER (WHERE rain_actual IS NOT NULL) AS mean_probability,
avg(power(probability - CASE WHEN rain_next_1h_actual THEN 1.0 ELSE 0.0 END, 2.0)) avg(CASE WHEN rain_actual THEN 1.0 ELSE 0.0 END) FILTER (WHERE rain_actual IS NOT NULL) AS observed_positive_rate,
FILTER (WHERE rain_next_1h_actual IS NOT NULL) AS brier_score, avg(power(probability - CASE WHEN rain_actual THEN 1.0 ELSE 0.0 END, 2.0))
FILTER (WHERE rain_actual IS NOT NULL) AS brier_score,
avg( avg(
CASE CASE
WHEN rain_next_1h_actual IS NULL THEN NULL WHEN rain_actual IS NULL THEN NULL
WHEN predict_rain = rain_next_1h_actual THEN 1.0 WHEN predict_rain = rain_actual THEN 1.0
ELSE 0.0 ELSE 0.0
END END
) AS decision_accuracy ) AS decision_accuracy
FROM predictions_rain_1h FROM all_predictions
GROUP BY 1,2,3,4; GROUP BY 1,2,3,4,5;
CREATE OR REPLACE VIEW rain_pipeline_health AS CREATE OR REPLACE VIEW rain_pipeline_health AS
WITH sites AS ( WITH prediction_latest AS (
SELECT
site,
max(generated_at) AS prediction_generated_latest_ts,
max(evaluated_at) AS prediction_evaluated_latest_ts
FROM (
SELECT site, generated_at, evaluated_at FROM predictions_rain_1h
UNION ALL
SELECT site, generated_at, evaluated_at FROM predictions_rain_4h
) p
GROUP BY site
),
sites AS (
SELECT DISTINCT site FROM observations_ws90 SELECT DISTINCT site FROM observations_ws90
UNION UNION
SELECT DISTINCT site FROM observations_baro SELECT DISTINCT site FROM observations_baro
@@ -145,12 +205,16 @@ WITH sites AS (
SELECT DISTINCT site FROM forecast_openmeteo_hourly SELECT DISTINCT site FROM forecast_openmeteo_hourly
UNION UNION
SELECT DISTINCT site FROM predictions_rain_1h SELECT DISTINCT site FROM predictions_rain_1h
UNION
SELECT DISTINCT site FROM predictions_rain_4h
) )
SELECT SELECT
s.site, s.site,
(SELECT max(ts) FROM observations_ws90 w WHERE w.site = s.site) AS ws90_latest_ts, (SELECT max(ts) FROM observations_ws90 w WHERE w.site = s.site) AS ws90_latest_ts,
(SELECT max(ts) FROM observations_baro b WHERE b.site = s.site) AS baro_latest_ts, (SELECT max(ts) FROM observations_baro b WHERE b.site = s.site) AS baro_latest_ts,
(SELECT max(ts) FROM forecast_openmeteo_hourly f WHERE f.site = s.site) AS forecast_latest_ts, (SELECT max(ts) FROM forecast_openmeteo_hourly f WHERE f.site = s.site) AS forecast_latest_ts,
(SELECT max(generated_at) FROM predictions_rain_1h p WHERE p.site = s.site) AS prediction_generated_latest_ts, p.prediction_generated_latest_ts,
(SELECT max(evaluated_at) FROM predictions_rain_1h p WHERE p.site = s.site) AS prediction_evaluated_latest_ts p.prediction_evaluated_latest_ts
FROM sites s; FROM sites s
LEFT JOIN prediction_latest p
ON p.site = s.site;
+29
View File
@@ -0,0 +1,29 @@
-- Add 4-hour rain prediction storage table.
-- Safe to re-run.
CREATE TABLE IF NOT EXISTS predictions_rain_4h (
ts TIMESTAMPTZ NOT NULL,
generated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
site TEXT NOT NULL,
model_name TEXT NOT NULL,
model_version TEXT NOT NULL,
threshold DOUBLE PRECISION NOT NULL,
probability DOUBLE PRECISION NOT NULL,
predict_rain BOOLEAN NOT NULL,
rain_next_4h_mm_actual DOUBLE PRECISION,
rain_next_4h_actual BOOLEAN,
evaluated_at TIMESTAMPTZ,
metadata JSONB,
PRIMARY KEY (site, model_name, model_version, ts)
);
SELECT create_hypertable('predictions_rain_4h', 'ts', if_not_exists => TRUE);
CREATE INDEX IF NOT EXISTS idx_predictions_rain_4h_site_ts
ON predictions_rain_4h(site, ts DESC);
CREATE INDEX IF NOT EXISTS idx_predictions_rain_4h_pending_eval
ON predictions_rain_4h(site, evaluated_at, ts DESC);
+3 -2
View File
@@ -34,8 +34,9 @@ services:
environment: environment:
DATABASE_URL: "postgres://postgres:postgres@timescaledb:5432/micrometeo?sslmode=disable" DATABASE_URL: "postgres://postgres:postgres@timescaledb:5432/micrometeo?sslmode=disable"
RAIN_SITE: "home" RAIN_SITE: "home"
RAIN_MODEL_NAME: "rain_next_1h" RAIN_HORIZON_HOURS: "4"
RAIN_MODEL_VERSION_BASE: "rain-auto-v1-extended" RAIN_MODEL_NAME: "rain_next_4h"
RAIN_MODEL_VERSION_BASE: "rain-auto-v2-extended-4h"
RAIN_MODEL_FAMILY: "auto" RAIN_MODEL_FAMILY: "auto"
RAIN_FEATURE_SET: "extended" RAIN_FEATURE_SET: "extended"
RAIN_FORECAST_MODEL: "ecmwf" RAIN_FORECAST_MODEL: "ecmwf"
+104 -5
View File
@@ -4,6 +4,14 @@ Operational guide for training, evaluating, deploying, monitoring, and rolling b
## 1) One-time Setup ## 1) One-time Setup
Apply 4-hour prediction table migration:
```sh
docker compose exec -T timescaledb \
psql -U postgres -d micrometeo \
-f /docker-entrypoint-initdb.d/003_rain_predictions_4h.sql
```
Apply monitoring views: Apply monitoring views:
```sh ```sh
@@ -21,6 +29,7 @@ python scripts/train_rain_model.py \
--site "home" \ --site "home" \
--start "2026-02-01T00:00:00Z" \ --start "2026-02-01T00:00:00Z" \
--end "2026-03-03T23:55:00Z" \ --end "2026-03-03T23:55:00Z" \
--horizon-hours 4 \
--feature-set "extended" \ --feature-set "extended" \
--model-family "auto" \ --model-family "auto" \
--forecast-model "ecmwf" \ --forecast-model "ecmwf" \
@@ -29,7 +38,7 @@ python scripts/train_rain_model.py \
--calibration-methods "none,sigmoid,isotonic" \ --calibration-methods "none,sigmoid,isotonic" \
--threshold-policy "walk_forward" \ --threshold-policy "walk_forward" \
--walk-forward-folds 4 \ --walk-forward-folds 4 \
--model-version "rain-auto-v1-extended" \ --model-version "rain-auto-v2-extended-4h" \
--out "models/rain_model.pkl" \ --out "models/rain_model.pkl" \
--report-out "models/rain_model_report.json" \ --report-out "models/rain_model_report.json" \
--model-card-out "models/model_card_{model_version}.md" \ --model-card-out "models/model_card_{model_version}.md" \
@@ -53,7 +62,8 @@ Review in report:
python scripts/predict_rain_model.py \ python scripts/predict_rain_model.py \
--site home \ --site home \
--model-path "models/rain_model.pkl" \ --model-path "models/rain_model.pkl" \
--model-name "rain_next_1h" \ --model-name "rain_next_4h" \
--horizon-hours 4 \
--dry-run --dry-run
``` ```
@@ -63,7 +73,8 @@ python scripts/predict_rain_model.py \
python scripts/predict_rain_model.py \ python scripts/predict_rain_model.py \
--site home \ --site home \
--model-path "models/rain_model.pkl" \ --model-path "models/rain_model.pkl" \
--model-name "rain_next_1h" --model-name "rain_next_4h" \
--horizon-hours 4
``` ```
## 4) Rollback ## 4) Rollback
@@ -72,6 +83,7 @@ python scripts/predict_rain_model.py \
2. If promotion fails or no candidate model is produced, the worker keeps the active model unchanged. 2. If promotion fails or no candidate model is produced, the worker keeps the active model unchanged.
3. If inference starts without `RAIN_MODEL_PATH` but backup exists, the worker restores from backup automatically. 3. If inference starts without `RAIN_MODEL_PATH` but backup exists, the worker restores from backup automatically.
4. Keep failed candidate artifacts for postmortem. 4. Keep failed candidate artifacts for postmortem.
5. During 4-hour rollout stabilization, keep `predictions_rain_1h` and `rain_next_1h` model artifacts available for immediate fallback.
## 5) Monitoring ## 5) Monitoring
@@ -118,12 +130,13 @@ Use the health-check script in cron, systemd timer, or your alerting scheduler:
```sh ```sh
python scripts/check_rain_pipeline_health.py \ python scripts/check_rain_pipeline_health.py \
--site home \ --site home \
--model-name rain_next_1h \ --model-name rain_next_4h \
--horizon-hours 4 \
--max-ws90-age 20m \ --max-ws90-age 20m \
--max-baro-age 30m \ --max-baro-age 30m \
--max-forecast-age 3h \ --max-forecast-age 3h \
--max-prediction-age 30m \ --max-prediction-age 30m \
--max-pending-eval-age 3h \ --max-pending-eval-age 6h \
--max-pending-eval-rows 200 --max-pending-eval-rows 200
``` ```
@@ -138,6 +151,7 @@ The script exits non-zero on failure, so it can directly drive alerting.
- `RAIN_THRESHOLD_POLICY` - `RAIN_THRESHOLD_POLICY`
- `RAIN_WALK_FORWARD_FOLDS` - `RAIN_WALK_FORWARD_FOLDS`
- `RAIN_ALLOW_EMPTY_DATA` - `RAIN_ALLOW_EMPTY_DATA`
- `RAIN_HORIZON_HOURS`
- `RAIN_MODEL_BACKUP_PATH` - `RAIN_MODEL_BACKUP_PATH`
- `RAIN_MODEL_CARD_PATH` - `RAIN_MODEL_CARD_PATH`
@@ -156,3 +170,88 @@ python scripts/recommend_rain_model.py \
--top-k 5 \ --top-k 5 \
--json-out "models/rain_model_recommendation.json" --json-out "models/rain_model_recommendation.json"
``` ```
## 9) Staged 4h Rollout Checklist
Run this sequence in production/staging to satisfy the 4h cutover gate:
1. Apply schema migration for 4h predictions:
```sh
docker compose exec -T timescaledb \
psql -U postgres -d micrometeo \
-f /docker-entrypoint-initdb.d/003_rain_predictions_4h.sql
```
2. Re-apply monitoring views (now include 1h + 4h unions):
```sh
docker compose exec -T timescaledb \
psql -U postgres -d micrometeo \
-f /docker-entrypoint-initdb.d/002_rain_monitoring_views.sql
```
3. Run a full 4h training/evaluation cycle and save report:
```sh
python scripts/train_rain_model.py \
--site "home" \
--start "2026-02-01T00:00:00Z" \
--end "2026-03-03T23:55:00Z" \
--horizon-hours 4 \
--feature-set "extended" \
--model-family "auto" \
--forecast-model "ecmwf" \
--tune-hyperparameters \
--threshold-policy "walk_forward" \
--walk-forward-folds 4 \
--model-version "rain-auto-v2-extended-4h" \
--out "models/rain_model_4h.pkl" \
--report-out "models/rain_model_report_4h.json"
```
4. Compare 4h metrics against the latest 1h benchmark report before switching dashboard defaults:
```sh
python scripts/compare_rain_reports.py \
--baseline "models/rain_model_report_1h.json" \
--candidate "models/rain_model_report_4h.json"
```
5. Run dry-run inference, then live inference with 4h model name/horizon:
```sh
python scripts/predict_rain_model.py \
--site home \
--model-path "models/rain_model_4h.pkl" \
--model-name "rain_next_4h" \
--horizon-hours 4 \
--dry-run
python scripts/predict_rain_model.py \
--site home \
--model-path "models/rain_model_4h.pkl" \
--model-name "rain_next_4h" \
--horizon-hours 4
```
6. Validate health checks and dashboard data path for 4h:
```sh
python scripts/check_rain_pipeline_health.py \
--site home \
--model-name rain_next_4h \
--horizon-hours 4 \
--max-pending-eval-age 6h
```
7. Keep 1h path live in parallel until 4h drift/calibration remains stable for at least 7 days.
### Fast rollback to 1h
If 4h performance or pipeline health regresses:
1. Set worker env back to:
`RAIN_HORIZON_HOURS=1`, `RAIN_MODEL_NAME=rain_next_1h`, and a known-good 1h model path/version.
2. Restart `rainml` service.
3. Confirm `check_rain_pipeline_health.py --horizon-hours 1 --model-name rain_next_1h` returns `ok`.
4. Keep `predictions_rain_4h` data for postmortem; do not drop tables during rollback.
+18 -7
View File
@@ -1,14 +1,14 @@
# Rain Prediction (Next 1 Hour) # Rain Prediction (Next 4 Hours)
This project includes a baseline workflow for **binary rain prediction**: This project includes a baseline workflow for **binary rain prediction**:
> **Will we see >= 0.2 mm of rain in the next hour?** > **Will we see >= 0.2 mm of rain in the next 4 hours?**
It uses local observations (WS90 + barometer), trains a logistic regression It uses local observations (WS90 + barometer), trains a logistic regression
baseline, and writes model-driven predictions back to TimescaleDB. baseline, and writes model-driven predictions back to TimescaleDB.
## P0 Decisions (Locked) ## P0 Decisions (Locked)
- Target: `rain_next_1h_mm >= 0.2`. - Target: `rain_next_4h_mm >= 0.2`.
- Primary use-case: low-noise rain heads-up signal for dashboard + alert candidate. - Primary use-case: low-noise rain heads-up signal for dashboard + alert candidate.
- Frozen v1 training window (UTC): `2026-02-01T00:00:00Z` to `2026-03-03T23:55:00Z`. - Frozen v1 training window (UTC): `2026-02-01T00:00:00Z` to `2026-03-03T23:55:00Z`.
- Threshold policy: choose threshold on validation set by maximizing recall under - Threshold policy: choose threshold on validation set by maximizing recall under
@@ -40,7 +40,7 @@ pip install -r scripts/requirements.txt
- `scripts/train_rain_model.py`: strict time-based split training and metrics report, with optional - `scripts/train_rain_model.py`: strict time-based split training and metrics report, with optional
validation-only hyperparameter tuning, calibration comparison, naive baseline comparison, and walk-forward folds. validation-only hyperparameter tuning, calibration comparison, naive baseline comparison, and walk-forward folds.
- `scripts/predict_rain_model.py`: inference using saved model artifact; upserts into - `scripts/predict_rain_model.py`: inference using saved model artifact; upserts into
`predictions_rain_1h`. `predictions_rain_4h`.
- `scripts/run_rain_ml_worker.py`: long-running worker for periodic training + prediction. - `scripts/run_rain_ml_worker.py`: long-running worker for periodic training + prediction.
- `scripts/check_rain_pipeline_health.py`: freshness/failure check for alerting. - `scripts/check_rain_pipeline_health.py`: freshness/failure check for alerting.
- `scripts/recommend_rain_model.py`: rank saved training reports and recommend a deployment candidate. - `scripts/recommend_rain_model.py`: rank saved training reports and recommend a deployment candidate.
@@ -60,7 +60,15 @@ Model-family options (`train_rain_model.py`):
## Usage ## Usage
### 1) Apply schema update (existing DBs) ### 1) Apply schema update (existing DBs)
`001_schema.sql` includes `predictions_rain_1h`. `003_rain_predictions_4h.sql` adds `predictions_rain_4h`.
```sh
docker compose exec -T timescaledb \
psql -U postgres -d micrometeo \
-f /docker-entrypoint-initdb.d/003_rain_predictions_4h.sql
```
`001_schema.sql` still remains safe to re-run for full schema parity.
```sh ```sh
docker compose exec -T timescaledb \ docker compose exec -T timescaledb \
@@ -76,6 +84,8 @@ docker compose exec -T timescaledb \
-f /docker-entrypoint-initdb.d/002_rain_monitoring_views.sql -f /docker-entrypoint-initdb.d/002_rain_monitoring_views.sql
``` ```
All examples below assume a 4-hour horizon (`--horizon-hours 4`) and `model-name=rain_next_4h`.
### 2) Run data audit ### 2) Run data audit
```sh ```sh
export DATABASE_URL="postgres://postgres:postgres@localhost:5432/micrometeo?sslmode=disable" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/micrometeo?sslmode=disable"
@@ -203,7 +213,8 @@ python scripts/train_rain_model.py \
python scripts/predict_rain_model.py \ python scripts/predict_rain_model.py \
--site home \ --site home \
--model-path "models/rain_model.pkl" \ --model-path "models/rain_model.pkl" \
--model-name "rain_next_1h" --model-name "rain_next_4h" \
--horizon-hours 4
``` ```
### 5) One-command P0 workflow ### 5) One-command P0 workflow
@@ -236,7 +247,7 @@ docker compose logs -f rainml
- Model card: `models/model_card_<model_version>.md` - Model card: `models/model_card_<model_version>.md`
- Model artifact: `models/rain_model.pkl` - Model artifact: `models/rain_model.pkl`
- Dataset snapshot: `models/datasets/rain_dataset_<model_version>_<feature_set>.csv` - Dataset snapshot: `models/datasets/rain_dataset_<model_version>_<feature_set>.csv`
- Prediction rows: `predictions_rain_1h` (probability + threshold decision + realized - Prediction rows: `predictions_rain_4h` (probability + threshold decision + realized
outcome fields once available) outcome fields once available)
### 7) Recommend deploy candidate from saved reports ### 7) Recommend deploy candidate from saved reports
+49 -19
View File
@@ -51,11 +51,14 @@ type RainPredictionPoint struct {
GeneratedAt time.Time `json:"generated_at"` GeneratedAt time.Time `json:"generated_at"`
ModelName string `json:"model_name"` ModelName string `json:"model_name"`
ModelVersion string `json:"model_version"` ModelVersion string `json:"model_version"`
HorizonHours int `json:"horizon_hours"`
Threshold float64 `json:"threshold"` Threshold float64 `json:"threshold"`
Probability float64 `json:"probability"` Probability float64 `json:"probability"`
PredictRain bool `json:"predict_rain"` PredictRain bool `json:"predict_rain"`
RainNext1hMM *float64 `json:"rain_next_1h_mm_actual,omitempty"` RainNextMM *float64 `json:"rain_next_mm_actual,omitempty"`
RainNext1hActual *bool `json:"rain_next_1h_actual,omitempty"` RainNextActual *bool `json:"rain_next_actual,omitempty"`
RainNext1hMM *float64 `json:"rain_next_1h_mm_actual,omitempty"` // backward-compatible alias
RainNext1hActual *bool `json:"rain_next_1h_actual,omitempty"` // backward-compatible alias
EvaluatedAt *time.Time `json:"evaluated_at,omitempty"` EvaluatedAt *time.Time `json:"evaluated_at,omitempty"`
} }
@@ -365,8 +368,24 @@ func (d *DB) ForecastSeriesRange(ctx context.Context, site, model string, start,
}, nil }, nil
} }
func (d *DB) LatestRainPrediction(ctx context.Context, site, modelName string) (*RainPredictionPoint, error) { func predictionStorageForHorizon(horizonHours int) (table string, mmCol string, flagCol string, err error) {
query := ` switch horizonHours {
case 1:
return "predictions_rain_1h", "rain_next_1h_mm_actual", "rain_next_1h_actual", nil
case 4:
return "predictions_rain_4h", "rain_next_4h_mm_actual", "rain_next_4h_actual", nil
default:
return "", "", "", fmt.Errorf("unsupported rain prediction horizon: %d", horizonHours)
}
}
func (d *DB) LatestRainPrediction(ctx context.Context, site, modelName string, horizonHours int) (*RainPredictionPoint, error) {
table, mmCol, flagCol, err := predictionStorageForHorizon(horizonHours)
if err != nil {
return nil, err
}
query := fmt.Sprintf(`
SELECT SELECT
ts, ts,
generated_at, generated_at,
@@ -375,15 +394,15 @@ func (d *DB) LatestRainPrediction(ctx context.Context, site, modelName string) (
threshold, threshold,
probability, probability,
predict_rain, predict_rain,
rain_next_1h_mm_actual, %s,
rain_next_1h_actual, %s,
evaluated_at evaluated_at
FROM predictions_rain_1h FROM %s
WHERE site = $1 WHERE site = $1
AND model_name = $2 AND model_name = $2
ORDER BY ts DESC, generated_at DESC ORDER BY ts DESC, generated_at DESC
LIMIT 1 LIMIT 1
` `, mmCol, flagCol, table)
var ( var (
p RainPredictionPoint p RainPredictionPoint
@@ -393,7 +412,7 @@ func (d *DB) LatestRainPrediction(ctx context.Context, site, modelName string) (
predictRain sql.NullBool predictRain sql.NullBool
) )
err := d.Pool.QueryRow(ctx, query, site, modelName).Scan( err = d.Pool.QueryRow(ctx, query, site, modelName).Scan(
&p.TS, &p.TS,
&p.GeneratedAt, &p.GeneratedAt,
&p.ModelName, &p.ModelName,
@@ -421,14 +440,22 @@ func (d *DB) LatestRainPrediction(ctx context.Context, site, modelName string) (
if predictRain.Valid { if predictRain.Valid {
p.PredictRain = predictRain.Bool p.PredictRain = predictRain.Bool
} }
p.RainNext1hMM = nullFloatPtr(rainMM) p.HorizonHours = horizonHours
p.RainNext1hActual = nullBoolPtr(rainActual) p.RainNextMM = nullFloatPtr(rainMM)
p.RainNextActual = nullBoolPtr(rainActual)
p.RainNext1hMM = p.RainNextMM
p.RainNext1hActual = p.RainNextActual
p.EvaluatedAt = nullTimePtr(evaluatedAt) p.EvaluatedAt = nullTimePtr(evaluatedAt)
return &p, nil return &p, nil
} }
func (d *DB) RainPredictionSeriesRange(ctx context.Context, site, modelName string, start, end time.Time) ([]RainPredictionPoint, error) { func (d *DB) RainPredictionSeriesRange(ctx context.Context, site, modelName string, horizonHours int, start, end time.Time) ([]RainPredictionPoint, error) {
query := ` table, mmCol, flagCol, err := predictionStorageForHorizon(horizonHours)
if err != nil {
return nil, err
}
query := fmt.Sprintf(`
SELECT DISTINCT ON (ts) SELECT DISTINCT ON (ts)
ts, ts,
generated_at, generated_at,
@@ -437,16 +464,16 @@ func (d *DB) RainPredictionSeriesRange(ctx context.Context, site, modelName stri
threshold, threshold,
probability, probability,
predict_rain, predict_rain,
rain_next_1h_mm_actual, %s,
rain_next_1h_actual, %s,
evaluated_at evaluated_at
FROM predictions_rain_1h FROM %s
WHERE site = $1 WHERE site = $1
AND model_name = $2 AND model_name = $2
AND ts >= $3 AND ts >= $3
AND ts <= $4 AND ts <= $4
ORDER BY ts ASC, generated_at DESC ORDER BY ts ASC, generated_at DESC
` `, mmCol, flagCol, table)
rows, err := d.Pool.Query(ctx, query, site, modelName, start, end) rows, err := d.Pool.Query(ctx, query, site, modelName, start, end)
if err != nil { if err != nil {
@@ -486,8 +513,11 @@ func (d *DB) RainPredictionSeriesRange(ctx context.Context, site, modelName stri
if predictRain.Valid { if predictRain.Valid {
p.PredictRain = predictRain.Bool p.PredictRain = predictRain.Bool
} }
p.RainNext1hMM = nullFloatPtr(rainMM) p.HorizonHours = horizonHours
p.RainNext1hActual = nullBoolPtr(rainActual) p.RainNextMM = nullFloatPtr(rainMM)
p.RainNextActual = nullBoolPtr(rainActual)
p.RainNext1hMM = p.RainNextMM
p.RainNext1hActual = p.RainNextActual
p.EvaluatedAt = nullTimePtr(evaluatedAt) p.EvaluatedAt = nullTimePtr(evaluatedAt)
points = append(points, p) points = append(points, p)
} }
+30 -8
View File
@@ -10,6 +10,7 @@ import psycopg2
from rain_model_common import ( from rain_model_common import (
AVAILABLE_FEATURE_SETS, AVAILABLE_FEATURE_SETS,
DEFAULT_HORIZON_HOURS,
RAIN_EVENT_THRESHOLD_MM, RAIN_EVENT_THRESHOLD_MM,
build_dataset, build_dataset,
feature_columns_for_set, feature_columns_for_set,
@@ -17,8 +18,11 @@ from rain_model_common import (
fetch_baro, fetch_baro,
fetch_forecast, fetch_forecast,
fetch_ws90, fetch_ws90,
normalize_horizon_hours,
model_frame, model_frame,
parse_time, parse_time,
rain_next_flag_col,
rain_next_mm_col,
to_builtin, to_builtin,
) )
@@ -29,6 +33,12 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--site", required=True, help="Site name (e.g. home).") parser.add_argument("--site", required=True, help="Site name (e.g. home).")
parser.add_argument("--start", help="Start time (RFC3339 or YYYY-MM-DD).") parser.add_argument("--start", help="Start time (RFC3339 or YYYY-MM-DD).")
parser.add_argument("--end", help="End time (RFC3339 or YYYY-MM-DD).") parser.add_argument("--end", help="End time (RFC3339 or YYYY-MM-DD).")
parser.add_argument(
"--horizon-hours",
type=int,
default=DEFAULT_HORIZON_HOURS,
help="Prediction horizon in hours for target/label auditing.",
)
parser.add_argument( parser.add_argument(
"--feature-set", "--feature-set",
default="baseline", default="baseline",
@@ -57,13 +67,13 @@ def longest_zero_run(counts: np.ndarray) -> int:
return best return best
def build_weekly_balance(model_df): def build_weekly_balance(model_df, target_col: str):
weekly = model_df.copy() weekly = model_df.copy()
iso = weekly.index.to_series().dt.isocalendar() iso = weekly.index.to_series().dt.isocalendar()
weekly["year_week"] = iso["year"].astype(str) + "-W" + iso["week"].astype(str).str.zfill(2) weekly["year_week"] = iso["year"].astype(str) + "-W" + iso["week"].astype(str).str.zfill(2)
grouped = ( grouped = (
weekly.groupby("year_week")["rain_next_1h"] weekly.groupby("year_week")[target_col]
.agg(total_rows="count", positive_rows="sum") .agg(total_rows="count", positive_rows="sum")
.reset_index() .reset_index()
.sort_values("year_week") .sort_values("year_week")
@@ -79,6 +89,9 @@ def main() -> int:
start = parse_time(args.start) if args.start else "" start = parse_time(args.start) if args.start else ""
end = parse_time(args.end) if args.end else "" end = parse_time(args.end) if args.end else ""
horizon_hours = normalize_horizon_hours(args.horizon_hours)
target_col = rain_next_flag_col(horizon_hours)
target_mm_col = rain_next_mm_col(horizon_hours)
feature_cols = feature_columns_for_set(args.feature_set) feature_cols = feature_columns_for_set(args.feature_set)
needs_forecast = feature_columns_need_forecast(feature_cols) needs_forecast = feature_columns_need_forecast(feature_cols)
@@ -89,8 +102,14 @@ def main() -> int:
if needs_forecast: if needs_forecast:
forecast = fetch_forecast(conn, args.site, start, end, model=args.forecast_model) forecast = fetch_forecast(conn, args.site, start, end, model=args.forecast_model)
df = build_dataset(ws90, baro, forecast=forecast, rain_event_threshold_mm=RAIN_EVENT_THRESHOLD_MM) df = build_dataset(
model_df = model_frame(df, feature_cols, require_target=True) ws90,
baro,
forecast=forecast,
rain_event_threshold_mm=RAIN_EVENT_THRESHOLD_MM,
horizon_hours=horizon_hours,
)
model_df = model_frame(df, feature_cols, require_target=True, target_col=target_col)
ws90_dupes = int(ws90.duplicated(subset=["ts", "station_id"]).sum()) if not ws90.empty else 0 ws90_dupes = int(ws90.duplicated(subset=["ts", "station_id"]).sum()) if not ws90.empty else 0
baro_dupes = int(baro.duplicated(subset=["ts", "source"]).sum()) if not baro.empty else 0 baro_dupes = int(baro.duplicated(subset=["ts", "source"]).sum()) if not baro.empty else 0
@@ -114,7 +133,7 @@ def main() -> int:
baro_max_gap_min = longest_zero_run(np.array(baro_counts)) * 5 if len(baro_counts) else 0 baro_max_gap_min = longest_zero_run(np.array(baro_counts)) * 5 if len(baro_counts) else 0
missingness = {} missingness = {}
for col in feature_cols + ["pressure_hpa", "rain_mm", "rain_inc", "rain_next_1h_mm"]: for col in feature_cols + ["pressure_hpa", "rain_mm", "rain_inc", target_mm_col]:
if col in df.columns: if col in df.columns:
missingness[col] = float(df[col].isna().mean()) missingness[col] = float(df[col].isna().mean())
@@ -125,9 +144,12 @@ def main() -> int:
report = { report = {
"site": args.site, "site": args.site,
"feature_set": args.feature_set, "feature_set": args.feature_set,
"horizon_hours": horizon_hours,
"target_column": target_col,
"target_mm_column": target_mm_col,
"feature_columns": feature_cols, "feature_columns": feature_cols,
"forecast_model": args.forecast_model if needs_forecast else None, "forecast_model": args.forecast_model if needs_forecast else None,
"target_definition": f"rain_next_1h_mm >= {RAIN_EVENT_THRESHOLD_MM:.2f}", "target_definition": f"{target_mm_col} >= {RAIN_EVENT_THRESHOLD_MM:.2f}",
"requested_window": { "requested_window": {
"start": start or None, "start": start or None,
"end": end or None, "end": end or None,
@@ -167,8 +189,8 @@ def main() -> int:
"max_rain_increment_5m_mm": max_rain_inc, "max_rain_increment_5m_mm": max_rain_inc,
}, },
"class_balance": { "class_balance": {
"overall_positive_rate": float(model_df["rain_next_1h"].mean()) if not model_df.empty else None, "overall_positive_rate": float(model_df[target_col].mean()) if not model_df.empty else None,
"weekly": build_weekly_balance(model_df) if not model_df.empty else [], "weekly": build_weekly_balance(model_df, target_col=target_col) if not model_df.empty else [],
}, },
} }
report = to_builtin(report) report = to_builtin(report)
+34 -6
View File
@@ -9,6 +9,24 @@ from typing import Any
import psycopg2 import psycopg2
DEFAULT_HORIZON_HOURS = 4
def normalize_horizon_hours(horizon_hours: int) -> int:
out = int(horizon_hours)
if out <= 0:
raise ValueError("horizon_hours must be > 0")
return out
def prediction_table_for_horizon(horizon_hours: int) -> str:
horizon = normalize_horizon_hours(horizon_hours)
if horizon == 1:
return "predictions_rain_1h"
if horizon == 4:
return "predictions_rain_4h"
raise ValueError(f"unsupported prediction-table horizon: {horizon_hours}")
def parse_duration(value: str) -> timedelta: def parse_duration(value: str) -> timedelta:
raw = value.strip().lower() raw = value.strip().lower()
@@ -27,14 +45,20 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Check freshness/health of rain-model data and predictions.") parser = argparse.ArgumentParser(description="Check freshness/health of rain-model data and predictions.")
parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.") parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.")
parser.add_argument("--site", required=True, help="Site name.") parser.add_argument("--site", required=True, help="Site name.")
parser.add_argument("--model-name", default="rain_next_1h", help="Prediction model_name to check.") parser.add_argument("--model-name", default="rain_next_4h", help="Prediction model_name to check.")
parser.add_argument(
"--horizon-hours",
type=int,
default=DEFAULT_HORIZON_HOURS,
help="Prediction horizon in hours used to select prediction storage table.",
)
parser.add_argument("--max-ws90-age", default="20m", help="Max allowed age for ws90 latest row.") parser.add_argument("--max-ws90-age", default="20m", help="Max allowed age for ws90 latest row.")
parser.add_argument("--max-baro-age", default="30m", help="Max allowed age for barometer latest row.") parser.add_argument("--max-baro-age", default="30m", help="Max allowed age for barometer latest row.")
parser.add_argument("--max-forecast-age", default="3h", help="Max allowed age for forecast latest row.") parser.add_argument("--max-forecast-age", default="3h", help="Max allowed age for forecast latest row.")
parser.add_argument("--max-prediction-age", default="30m", help="Max allowed age for latest prediction write.") parser.add_argument("--max-prediction-age", default="30m", help="Max allowed age for latest prediction write.")
parser.add_argument( parser.add_argument(
"--max-pending-eval-age", "--max-pending-eval-age",
default="3h", default="6h",
help="Pending evaluations older than this count toward alert.", help="Pending evaluations older than this count toward alert.",
) )
parser.add_argument( parser.add_argument(
@@ -91,6 +115,8 @@ def main() -> int:
max_forecast_age = parse_duration(args.max_forecast_age) max_forecast_age = parse_duration(args.max_forecast_age)
max_prediction_age = parse_duration(args.max_prediction_age) max_prediction_age = parse_duration(args.max_prediction_age)
max_pending_eval_age = parse_duration(args.max_pending_eval_age) max_pending_eval_age = parse_duration(args.max_pending_eval_age)
horizon_hours = normalize_horizon_hours(args.horizon_hours)
prediction_table = prediction_table_for_horizon(horizon_hours)
with psycopg2.connect(args.db_url) as conn: with psycopg2.connect(args.db_url) as conn:
ws90_latest = fetch_latest_ts( ws90_latest = fetch_latest_ts(
@@ -110,9 +136,9 @@ def main() -> int:
) )
prediction_latest = fetch_latest_ts( prediction_latest = fetch_latest_ts(
conn, conn,
""" f"""
SELECT max(generated_at) SELECT max(generated_at)
FROM predictions_rain_1h FROM {prediction_table}
WHERE site = %s WHERE site = %s
AND model_name = %s AND model_name = %s
""", """,
@@ -120,9 +146,9 @@ def main() -> int:
) )
pending_eval_rows = fetch_count( pending_eval_rows = fetch_count(
conn, conn,
""" f"""
SELECT count(*) SELECT count(*)
FROM predictions_rain_1h FROM {prediction_table}
WHERE site = %s WHERE site = %s
AND model_name = %s AND model_name = %s
AND evaluated_at IS NULL AND evaluated_at IS NULL
@@ -188,6 +214,8 @@ def main() -> int:
"generated_at": now.isoformat(), "generated_at": now.isoformat(),
"site": args.site, "site": args.site,
"model_name": args.model_name, "model_name": args.model_name,
"horizon_hours": horizon_hours,
"prediction_table": prediction_table,
"status": overall_status, "status": overall_status,
"failing_checks": failing, "failing_checks": failing,
"checks": checks, "checks": checks,
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Compare two rain-model training reports.")
parser.add_argument("--baseline", required=True, help="Baseline report path (for example 1h).")
parser.add_argument("--candidate", required=True, help="Candidate report path (for example 4h).")
return parser.parse_args()
def load_json(path: str) -> dict[str, Any]:
p = Path(path)
with p.open("r", encoding="utf-8") as f:
return json.load(f)
def to_float(v: Any) -> float | None:
if v is None:
return None
try:
return float(v)
except (TypeError, ValueError):
return None
def metric(report: dict[str, Any], split: str, key: str) -> float | None:
return to_float(report.get(split, {}).get(key))
def delta_str(base: float | None, cand: float | None) -> str:
if base is None or cand is None:
return "n/a"
d = cand - base
return f"{d:+.4f}"
def main() -> int:
args = parse_args()
baseline = load_json(args.baseline)
candidate = load_json(args.candidate)
pairs = [
("precision", metric(baseline, "test_metrics", "precision"), metric(candidate, "test_metrics", "precision")),
("recall", metric(baseline, "test_metrics", "recall"), metric(candidate, "test_metrics", "recall")),
("f1", metric(baseline, "test_metrics", "f1"), metric(candidate, "test_metrics", "f1")),
("pr_auc", metric(baseline, "test_metrics", "pr_auc"), metric(candidate, "test_metrics", "pr_auc")),
("roc_auc", metric(baseline, "test_metrics", "roc_auc"), metric(candidate, "test_metrics", "roc_auc")),
("brier", metric(baseline, "test_metrics", "brier"), metric(candidate, "test_metrics", "brier")),
]
print("Rain report comparison:")
print(
f" baseline: version={baseline.get('model_version')} "
f"horizon={baseline.get('horizon_hours')}h "
f"target={baseline.get('target_definition')}"
)
print(
f" candidate: version={candidate.get('model_version')} "
f"horizon={candidate.get('horizon_hours')}h "
f"target={candidate.get('target_definition')}"
)
print(" metrics (candidate - baseline):")
for name, base, cand in pairs:
base_txt = "n/a" if base is None else f"{base:.4f}"
cand_txt = "n/a" if cand is None else f"{cand:.4f}"
print(f" {name}: baseline={base_txt} candidate={cand_txt} delta={delta_str(base, cand)}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+45 -17
View File
@@ -9,13 +9,18 @@ import psycopg2
from psycopg2.extras import Json from psycopg2.extras import Json
from rain_model_common import ( from rain_model_common import (
DEFAULT_HORIZON_HOURS,
build_dataset, build_dataset,
feature_columns_need_forecast, feature_columns_need_forecast,
fetch_baro, fetch_baro,
fetch_forecast, fetch_forecast,
fetch_ws90, fetch_ws90,
model_frame, model_frame,
normalize_horizon_hours,
parse_time, parse_time,
prediction_table_for_horizon,
rain_next_flag_col,
rain_next_mm_col,
to_builtin, to_builtin,
) )
@@ -30,8 +35,13 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.") parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.")
parser.add_argument("--site", required=True, help="Site name (e.g. home).") parser.add_argument("--site", required=True, help="Site name (e.g. home).")
parser.add_argument("--model-path", default="models/rain_model.pkl", help="Path to trained model artifact.") parser.add_argument("--model-path", default="models/rain_model.pkl", help="Path to trained model artifact.")
parser.add_argument("--model-name", default="rain_next_1h", help="Logical prediction model name.") parser.add_argument("--model-name", default="rain_next_4h", help="Logical prediction model name.")
parser.add_argument("--model-version", help="Override artifact model_version.") parser.add_argument("--model-version", help="Override artifact model_version.")
parser.add_argument(
"--horizon-hours",
type=int,
help="Prediction horizon in hours. Defaults to artifact horizon when present, else 4.",
)
parser.add_argument( parser.add_argument(
"--at", "--at",
help="Prediction timestamp (RFC3339 or YYYY-MM-DD). Default: current UTC time.", help="Prediction timestamp (RFC3339 or YYYY-MM-DD). Default: current UTC time.",
@@ -98,9 +108,21 @@ def main() -> int:
threshold = float(artifact.get("threshold", 0.5)) threshold = float(artifact.get("threshold", 0.5))
model_version = args.model_version or artifact.get("model_version") or "unknown" model_version = args.model_version or artifact.get("model_version") or "unknown"
forecast_model = str(artifact.get("forecast_model") or args.forecast_model) forecast_model = str(artifact.get("forecast_model") or args.forecast_model)
artifact_horizon = artifact.get("horizon_hours")
if args.horizon_hours is not None:
horizon_hours = normalize_horizon_hours(args.horizon_hours)
elif artifact_horizon is not None:
horizon_hours = normalize_horizon_hours(int(artifact_horizon))
else:
horizon_hours = DEFAULT_HORIZON_HOURS
target_col = str(artifact.get("target_col") or rain_next_flag_col(horizon_hours))
target_mm_col = str(artifact.get("target_mm_col") or rain_next_mm_col(horizon_hours))
prediction_table = prediction_table_for_horizon(horizon_hours)
actual_mm_col = f"{target_mm_col}_actual"
actual_flag_col = f"{target_col}_actual"
fetch_start = (at - timedelta(hours=args.history_hours)).isoformat() fetch_start = (at - timedelta(hours=args.history_hours)).isoformat()
fetch_end = (at + timedelta(hours=1, minutes=5)).isoformat() fetch_end = (at + timedelta(hours=horizon_hours, minutes=5)).isoformat()
with psycopg2.connect(args.db_url) as conn: with psycopg2.connect(args.db_url) as conn:
ws90 = fetch_ws90(conn, args.site, fetch_start, fetch_end) ws90 = fetch_ws90(conn, args.site, fetch_start, fetch_end)
@@ -122,7 +144,7 @@ def main() -> int:
return 0 return 0
raise RuntimeError(message) raise RuntimeError(message)
full_df = build_dataset(ws90, baro, forecast=forecast) full_df = build_dataset(ws90, baro, forecast=forecast, horizon_hours=horizon_hours)
feature_df = model_frame(full_df, feature_cols=features, require_target=False) feature_df = model_frame(full_df, feature_cols=features, require_target=False)
candidates = feature_df.loc[feature_df.index <= at] candidates = feature_df.loc[feature_df.index <= at]
if candidates.empty: if candidates.empty:
@@ -143,9 +165,9 @@ def main() -> int:
actual_flag = None actual_flag = None
evaluated_at = None evaluated_at = None
latest_available = full_df.index.max().to_pydatetime() latest_available = full_df.index.max().to_pydatetime()
if pred_ts + timedelta(hours=1) <= latest_available: if pred_ts + timedelta(hours=horizon_hours) <= latest_available:
next_mm = full_df.loc[pred_ts, "rain_next_1h_mm"] next_mm = full_df.loc[pred_ts, target_mm_col]
next_flag = full_df.loc[pred_ts, "rain_next_1h"] next_flag = full_df.loc[pred_ts, target_col]
if next_mm == next_mm: # NaN-safe check if next_mm == next_mm: # NaN-safe check
actual_mm = float(next_mm) actual_mm = float(next_mm)
if next_flag == next_flag: if next_flag == next_flag:
@@ -156,6 +178,10 @@ def main() -> int:
"artifact_path": args.model_path, "artifact_path": args.model_path,
"artifact_model_version": artifact.get("model_version"), "artifact_model_version": artifact.get("model_version"),
"artifact_feature_set": feature_set, "artifact_feature_set": feature_set,
"horizon_hours": horizon_hours,
"target_col": target_col,
"target_mm_col": target_mm_col,
"prediction_table": prediction_table,
"forecast_model": forecast_model if needs_forecast else None, "forecast_model": forecast_model if needs_forecast else None,
"needs_forecast_features": needs_forecast, "needs_forecast_features": needs_forecast,
"feature_values": {col: float(row.iloc[0][col]) for col in features}, "feature_values": {col: float(row.iloc[0][col]) for col in features},
@@ -170,6 +196,7 @@ def main() -> int:
print(f" site: {args.site}") print(f" site: {args.site}")
print(f" model_name: {args.model_name}") print(f" model_name: {args.model_name}")
print(f" model_version: {model_version}") print(f" model_version: {model_version}")
print(f" horizon_hours: {horizon_hours}")
if feature_set: if feature_set:
print(f" feature_set: {feature_set}") print(f" feature_set: {feature_set}")
print(f" pred_ts: {pred_ts.isoformat()}") print(f" pred_ts: {pred_ts.isoformat()}")
@@ -182,10 +209,8 @@ def main() -> int:
print("dry-run enabled; skipping DB upsert.") print("dry-run enabled; skipping DB upsert.")
return 0 return 0
with conn.cursor() as cur: query = f"""
cur.execute( INSERT INTO {prediction_table} (
"""
INSERT INTO predictions_rain_1h (
ts, ts,
generated_at, generated_at,
site, site,
@@ -194,8 +219,8 @@ def main() -> int:
threshold, threshold,
probability, probability,
predict_rain, predict_rain,
rain_next_1h_mm_actual, {actual_mm_col},
rain_next_1h_actual, {actual_flag_col},
evaluated_at, evaluated_at,
metadata metadata
) VALUES ( ) VALUES (
@@ -207,11 +232,14 @@ def main() -> int:
threshold = EXCLUDED.threshold, threshold = EXCLUDED.threshold,
probability = EXCLUDED.probability, probability = EXCLUDED.probability,
predict_rain = EXCLUDED.predict_rain, predict_rain = EXCLUDED.predict_rain,
rain_next_1h_mm_actual = COALESCE(EXCLUDED.rain_next_1h_mm_actual, predictions_rain_1h.rain_next_1h_mm_actual), {actual_mm_col} = COALESCE(EXCLUDED.{actual_mm_col}, {prediction_table}.{actual_mm_col}),
rain_next_1h_actual = COALESCE(EXCLUDED.rain_next_1h_actual, predictions_rain_1h.rain_next_1h_actual), {actual_flag_col} = COALESCE(EXCLUDED.{actual_flag_col}, {prediction_table}.{actual_flag_col}),
evaluated_at = COALESCE(EXCLUDED.evaluated_at, predictions_rain_1h.evaluated_at), evaluated_at = COALESCE(EXCLUDED.evaluated_at, {prediction_table}.evaluated_at),
metadata = EXCLUDED.metadata metadata = EXCLUDED.metadata
""", """
with conn.cursor() as cur:
cur.execute(
query,
( (
pred_ts, pred_ts,
args.site, args.site,
@@ -227,7 +255,7 @@ def main() -> int:
), ),
) )
conn.commit() conn.commit()
print("Prediction upserted into predictions_rain_1h.") print(f"Prediction upserted into {prediction_table}.")
return 0 return 0
+69 -16
View File
@@ -86,7 +86,46 @@ FEATURE_COLUMNS = BASELINE_FEATURE_COLUMNS
RAIN_EVENT_THRESHOLD_MM = 0.2 RAIN_EVENT_THRESHOLD_MM = 0.2
RAIN_SPIKE_THRESHOLD_MM_5M = 5.0 RAIN_SPIKE_THRESHOLD_MM_5M = 5.0
RAIN_HORIZON_BUCKETS = 12 # 12 * 5m = 1h BUCKET_MINUTES = 5
DEFAULT_HORIZON_HOURS = 4
SUPPORTED_PREDICTION_HORIZONS = (1, 4)
def normalize_horizon_hours(horizon_hours: int) -> int:
out = int(horizon_hours)
if out <= 0:
raise ValueError("horizon_hours must be > 0")
return out
def horizon_suffix(horizon_hours: int) -> str:
return f"{normalize_horizon_hours(horizon_hours)}h"
def horizon_buckets(horizon_hours: int) -> int:
hours = normalize_horizon_hours(horizon_hours)
return (hours * 60) // BUCKET_MINUTES
def rain_last_mm_col(horizon_hours: int) -> str:
return f"rain_last_{horizon_suffix(horizon_hours)}_mm"
def rain_next_mm_col(horizon_hours: int) -> str:
return f"rain_next_{horizon_suffix(horizon_hours)}_mm"
def rain_next_flag_col(horizon_hours: int) -> str:
return f"rain_next_{horizon_suffix(horizon_hours)}"
def prediction_table_for_horizon(horizon_hours: int) -> str:
horizon = normalize_horizon_hours(horizon_hours)
if horizon == 1:
return "predictions_rain_1h"
if horizon == 4:
return "predictions_rain_4h"
raise ValueError(f"unsupported prediction-table horizon: {horizon_hours}")
def parse_time(value: str) -> str: def parse_time(value: str) -> str:
@@ -232,6 +271,7 @@ def build_dataset(
baro: pd.DataFrame, baro: pd.DataFrame,
forecast: pd.DataFrame | None = None, forecast: pd.DataFrame | None = None,
rain_event_threshold_mm: float = RAIN_EVENT_THRESHOLD_MM, rain_event_threshold_mm: float = RAIN_EVENT_THRESHOLD_MM,
horizon_hours: int = 1,
) -> pd.DataFrame: ) -> pd.DataFrame:
if ws90.empty: if ws90.empty:
raise RuntimeError("no ws90 observations found") raise RuntimeError("no ws90 observations found")
@@ -261,12 +301,20 @@ def build_dataset(
df["rain_inc"] = df["rain_inc_raw"].clip(lower=0) df["rain_inc"] = df["rain_inc_raw"].clip(lower=0)
df["rain_spike_5m"] = df["rain_inc"] >= RAIN_SPIKE_THRESHOLD_MM_5M df["rain_spike_5m"] = df["rain_inc"] >= RAIN_SPIKE_THRESHOLD_MM_5M
window = RAIN_HORIZON_BUCKETS windows: dict[int, int] = {
df["rain_last_1h_mm"] = df["rain_inc"].rolling(window=window, min_periods=1).sum() 1: horizon_buckets(1),
df["rain_next_1h_mm"] = df["rain_inc"].rolling(window=window, min_periods=1).sum().shift(-(window - 1)) normalize_horizon_hours(horizon_hours): horizon_buckets(horizon_hours),
df["rain_next_1h"] = df["rain_next_1h_mm"] >= rain_event_threshold_mm }
for hours, window in windows.items():
rain_last_col = rain_last_mm_col(hours)
rain_next_mm = rain_next_mm_col(hours)
rain_next_flag = rain_next_flag_col(hours)
df[rain_last_col] = df["rain_inc"].rolling(window=window, min_periods=1).sum()
df[rain_next_mm] = df["rain_inc"].rolling(window=window, min_periods=1).sum().shift(-(window - 1))
df[rain_next_flag] = df[rain_next_mm] >= rain_event_threshold_mm
df["pressure_trend_1h"] = df["pressure_hpa"] - df["pressure_hpa"].shift(window) window_1h = horizon_buckets(1)
df["pressure_trend_1h"] = df["pressure_hpa"] - df["pressure_hpa"].shift(window_1h)
# Wind direction cyclical encoding. # Wind direction cyclical encoding.
radians = np.deg2rad(df["wind_dir_deg"] % 360.0) radians = np.deg2rad(df["wind_dir_deg"] % 360.0)
@@ -279,14 +327,14 @@ def build_dataset(
df["wind_avg_lag_5m"] = df["wind_avg_m_s"].shift(1) df["wind_avg_lag_5m"] = df["wind_avg_m_s"].shift(1)
df["pressure_lag_5m"] = df["pressure_hpa"].shift(1) df["pressure_lag_5m"] = df["pressure_hpa"].shift(1)
df["temp_roll_1h_mean"] = df["temperature_c"].rolling(window=window, min_periods=3).mean() df["temp_roll_1h_mean"] = df["temperature_c"].rolling(window=window_1h, min_periods=3).mean()
df["temp_roll_1h_std"] = df["temperature_c"].rolling(window=window, min_periods=3).std() df["temp_roll_1h_std"] = df["temperature_c"].rolling(window=window_1h, min_periods=3).std()
df["humidity_roll_1h_mean"] = df["humidity"].rolling(window=window, min_periods=3).mean() df["humidity_roll_1h_mean"] = df["humidity"].rolling(window=window_1h, min_periods=3).mean()
df["humidity_roll_1h_std"] = df["humidity"].rolling(window=window, min_periods=3).std() df["humidity_roll_1h_std"] = df["humidity"].rolling(window=window_1h, min_periods=3).std()
df["wind_avg_roll_1h_mean"] = df["wind_avg_m_s"].rolling(window=window, min_periods=3).mean() df["wind_avg_roll_1h_mean"] = df["wind_avg_m_s"].rolling(window=window_1h, min_periods=3).mean()
df["wind_gust_roll_1h_max"] = df["wind_max_m_s"].rolling(window=window, min_periods=3).max() df["wind_gust_roll_1h_max"] = df["wind_max_m_s"].rolling(window=window_1h, min_periods=3).max()
df["pressure_roll_1h_mean"] = df["pressure_hpa"].rolling(window=window, min_periods=3).mean() df["pressure_roll_1h_mean"] = df["pressure_hpa"].rolling(window=window_1h, min_periods=3).mean()
df["pressure_roll_1h_std"] = df["pressure_hpa"].rolling(window=window, min_periods=3).std() df["pressure_roll_1h_std"] = df["pressure_hpa"].rolling(window=window_1h, min_periods=3).std()
# Calendar/seasonality features (UTC based). # Calendar/seasonality features (UTC based).
hour_of_day = df.index.hour + (df.index.minute / 60.0) hour_of_day = df.index.hour + (df.index.minute / 60.0)
@@ -304,11 +352,16 @@ def build_dataset(
return df return df
def model_frame(df: pd.DataFrame, feature_cols: list[str] | None = None, require_target: bool = True) -> pd.DataFrame: def model_frame(
df: pd.DataFrame,
feature_cols: list[str] | None = None,
require_target: bool = True,
target_col: str | None = None,
) -> pd.DataFrame:
features = feature_cols or FEATURE_COLUMNS features = feature_cols or FEATURE_COLUMNS
required = list(features) required = list(features)
if require_target: if require_target:
required.append("rain_next_1h") required.append(target_col or rain_next_flag_col(1))
out = df.dropna(subset=required).copy() out = df.dropna(subset=required).copy()
return out.sort_index() return out.sort_index()
+7 -2
View File
@@ -4,7 +4,9 @@ set -euo pipefail
SITE="${SITE:-home}" SITE="${SITE:-home}"
START="${START:-2026-02-01T00:00:00Z}" START="${START:-2026-02-01T00:00:00Z}"
END="${END:-2026-03-03T23:55:00Z}" END="${END:-2026-03-03T23:55:00Z}"
MODEL_VERSION="${MODEL_VERSION:-rain-logreg-v1}" HORIZON_HOURS="${HORIZON_HOURS:-4}"
MODEL_NAME="${MODEL_NAME:-rain_next_${HORIZON_HOURS}h}"
MODEL_VERSION="${MODEL_VERSION:-rain-logreg-v2-${HORIZON_HOURS}h}"
MODEL_PATH="${MODEL_PATH:-models/rain_model.pkl}" MODEL_PATH="${MODEL_PATH:-models/rain_model.pkl}"
REPORT_PATH="${REPORT_PATH:-models/rain_model_report.json}" REPORT_PATH="${REPORT_PATH:-models/rain_model_report.json}"
AUDIT_PATH="${AUDIT_PATH:-models/rain_data_audit.json}" AUDIT_PATH="${AUDIT_PATH:-models/rain_data_audit.json}"
@@ -24,6 +26,7 @@ python scripts/audit_rain_data.py \
--site "$SITE" \ --site "$SITE" \
--start "$START" \ --start "$START" \
--end "$END" \ --end "$END" \
--horizon-hours "$HORIZON_HOURS" \
--feature-set "$FEATURE_SET" \ --feature-set "$FEATURE_SET" \
--forecast-model "$FORECAST_MODEL" \ --forecast-model "$FORECAST_MODEL" \
--out "$AUDIT_PATH" --out "$AUDIT_PATH"
@@ -33,6 +36,7 @@ python scripts/train_rain_model.py \
--site "$SITE" \ --site "$SITE" \
--start "$START" \ --start "$START" \
--end "$END" \ --end "$END" \
--horizon-hours "$HORIZON_HOURS" \
--train-ratio 0.7 \ --train-ratio 0.7 \
--val-ratio 0.15 \ --val-ratio 0.15 \
--min-precision 0.70 \ --min-precision 0.70 \
@@ -50,7 +54,8 @@ echo "Writing current prediction..."
python scripts/predict_rain_model.py \ python scripts/predict_rain_model.py \
--site "$SITE" \ --site "$SITE" \
--model-path "$MODEL_PATH" \ --model-path "$MODEL_PATH" \
--model-name "rain_next_1h" \ --model-name "$MODEL_NAME" \
--horizon-hours "$HORIZON_HOURS" \
--forecast-model "$FORECAST_MODEL" --forecast-model "$FORECAST_MODEL"
echo "P0 rain workflow complete." echo "P0 rain workflow complete."
+11 -2
View File
@@ -40,6 +40,7 @@ def read_env_bool(name: str, default: bool) -> bool:
class WorkerConfig: class WorkerConfig:
database_url: str database_url: str
site: str site: str
horizon_hours: int
model_name: str model_name: str
model_version_base: str model_version_base: str
model_family: str model_family: str
@@ -166,6 +167,8 @@ def run_training_cycle(cfg: WorkerConfig, env: dict[str, str]) -> None:
start, start,
"--end", "--end",
end, end,
"--horizon-hours",
str(cfg.horizon_hours),
"--feature-set", "--feature-set",
cfg.feature_set, cfg.feature_set,
"--forecast-model", "--forecast-model",
@@ -185,6 +188,8 @@ def run_training_cycle(cfg: WorkerConfig, env: dict[str, str]) -> None:
start, start,
"--end", "--end",
end, end,
"--horizon-hours",
str(cfg.horizon_hours),
"--train-ratio", "--train-ratio",
str(cfg.train_ratio), str(cfg.train_ratio),
"--val-ratio", "--val-ratio",
@@ -269,6 +274,8 @@ def run_predict_once(cfg: WorkerConfig, env: dict[str, str]) -> None:
str(cfg.model_path), str(cfg.model_path),
"--model-name", "--model-name",
cfg.model_name, cfg.model_name,
"--horizon-hours",
str(cfg.horizon_hours),
"--forecast-model", "--forecast-model",
cfg.forecast_model, cfg.forecast_model,
*(["--allow-empty"] if cfg.allow_empty_data else ["--strict-source-data"]), *(["--allow-empty"] if cfg.allow_empty_data else ["--strict-source-data"]),
@@ -289,8 +296,9 @@ def load_config() -> WorkerConfig:
return WorkerConfig( return WorkerConfig(
database_url=database_url, database_url=database_url,
site=read_env("RAIN_SITE", "home"), site=read_env("RAIN_SITE", "home"),
model_name=read_env("RAIN_MODEL_NAME", "rain_next_1h"), horizon_hours=read_env_int("RAIN_HORIZON_HOURS", 4),
model_version_base=read_env("RAIN_MODEL_VERSION_BASE", "rain-auto-v1-extended"), model_name=read_env("RAIN_MODEL_NAME", "rain_next_4h"),
model_version_base=read_env("RAIN_MODEL_VERSION_BASE", "rain-auto-v2-extended-4h"),
model_family=read_env("RAIN_MODEL_FAMILY", "auto"), model_family=read_env("RAIN_MODEL_FAMILY", "auto"),
feature_set=read_env("RAIN_FEATURE_SET", "extended"), feature_set=read_env("RAIN_FEATURE_SET", "extended"),
forecast_model=read_env("RAIN_FORECAST_MODEL", "ecmwf"), forecast_model=read_env("RAIN_FORECAST_MODEL", "ecmwf"),
@@ -338,6 +346,7 @@ def main() -> int:
print( print(
"[rain-ml] worker start " "[rain-ml] worker start "
f"site={cfg.site} " f"site={cfg.site} "
f"horizon_hours={cfg.horizon_hours} "
f"model_name={cfg.model_name} " f"model_name={cfg.model_name} "
f"model_family={cfg.model_family} " f"model_family={cfg.model_family} "
f"feature_set={cfg.feature_set} " f"feature_set={cfg.feature_set} "
+83 -24
View File
@@ -20,6 +20,7 @@ from sklearn.preprocessing import StandardScaler
from rain_model_common import ( from rain_model_common import (
AVAILABLE_FEATURE_SETS, AVAILABLE_FEATURE_SETS,
DEFAULT_HORIZON_HOURS,
RAIN_EVENT_THRESHOLD_MM, RAIN_EVENT_THRESHOLD_MM,
build_dataset, build_dataset,
evaluate_probs, evaluate_probs,
@@ -28,8 +29,13 @@ from rain_model_common import (
fetch_ws90, fetch_ws90,
feature_columns_for_set, feature_columns_for_set,
feature_columns_need_forecast, feature_columns_need_forecast,
horizon_suffix,
model_frame, model_frame,
normalize_horizon_hours,
parse_time, parse_time,
rain_last_mm_col,
rain_next_flag_col,
rain_next_mm_col,
safe_pr_auc, safe_pr_auc,
safe_roc_auc, safe_roc_auc,
select_threshold, select_threshold,
@@ -49,11 +55,17 @@ THRESHOLD_POLICIES = ("validation", "walk_forward")
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Train a rain prediction model (next 1h >= 0.2mm).") parser = argparse.ArgumentParser(description="Train a rain prediction model (next Nh >= threshold).")
parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.") parser.add_argument("--db-url", default=os.getenv("DATABASE_URL"), help="Postgres connection string.")
parser.add_argument("--site", required=True, help="Site name (e.g. home).") parser.add_argument("--site", required=True, help="Site name (e.g. home).")
parser.add_argument("--start", help="Start time (RFC3339 or YYYY-MM-DD).") parser.add_argument("--start", help="Start time (RFC3339 or YYYY-MM-DD).")
parser.add_argument("--end", help="End time (RFC3339 or YYYY-MM-DD).") parser.add_argument("--end", help="End time (RFC3339 or YYYY-MM-DD).")
parser.add_argument(
"--horizon-hours",
type=int,
default=DEFAULT_HORIZON_HOURS,
help="Prediction horizon in hours (for example 1 or 4).",
)
parser.add_argument("--train-ratio", type=float, default=0.7, help="Time-ordered train split ratio.") parser.add_argument("--train-ratio", type=float, default=0.7, help="Time-ordered train split ratio.")
parser.add_argument("--val-ratio", type=float, default=0.15, help="Time-ordered validation split ratio.") parser.add_argument("--val-ratio", type=float, default=0.15, help="Time-ordered validation split ratio.")
parser.add_argument( parser.add_argument(
@@ -464,14 +476,18 @@ def evaluate_calibration_methods(
return selected, results return selected, results
def evaluate_naive_baselines(test_df, y_test: np.ndarray) -> dict[str, Any]: def evaluate_naive_baselines(
test_df,
y_test: np.ndarray,
persistence_context_col: str,
) -> dict[str, Any]:
out: dict[str, Any] = {} out: dict[str, Any] = {}
if "rain_last_1h_mm" in test_df.columns: if persistence_context_col in test_df.columns:
rain_last = test_df["rain_last_1h_mm"].to_numpy(dtype=float) rain_last = test_df[persistence_context_col].to_numpy(dtype=float)
persistence_prob = (rain_last >= RAIN_EVENT_THRESHOLD_MM).astype(float) persistence_prob = (rain_last >= RAIN_EVENT_THRESHOLD_MM).astype(float)
out["persistence_last_1h"] = { out[f"persistence_{persistence_context_col}"] = {
"rule": f"predict rain when rain_last_1h_mm >= {RAIN_EVENT_THRESHOLD_MM:.2f}", "rule": f"predict rain when {persistence_context_col} >= {RAIN_EVENT_THRESHOLD_MM:.2f}",
"metrics": evaluate_probs(y_true=y_test, y_prob=persistence_prob, threshold=0.5), "metrics": evaluate_probs(y_true=y_test, y_prob=persistence_prob, threshold=0.5),
} }
@@ -512,6 +528,8 @@ def evaluate_sliced_performance(
y_true: np.ndarray, y_true: np.ndarray,
y_prob: np.ndarray, y_prob: np.ndarray,
threshold: float, threshold: float,
context_col: str,
context_label: str,
min_rows_per_slice: int = 30, min_rows_per_slice: int = 30,
) -> dict[str, Any]: ) -> dict[str, Any]:
frame = pd.DataFrame( frame = pd.DataFrame(
@@ -530,7 +548,11 @@ def evaluate_sliced_performance(
weekly_positive_rate = frame.groupby(week_label)["y_true"].transform("mean") weekly_positive_rate = frame.groupby(week_label)["y_true"].transform("mean")
rainy_week = weekly_positive_rate >= overall_rate rainy_week = weekly_positive_rate >= overall_rate
rain_context = test_df["rain_last_1h_mm"].to_numpy(dtype=float) if "rain_last_1h_mm" in test_df.columns else np.zeros(len(test_df)) rain_context = (
test_df[context_col].to_numpy(dtype=float)
if context_col in test_df.columns
else np.zeros(len(test_df))
)
wet_context = rain_context >= RAIN_EVENT_THRESHOLD_MM wet_context = rain_context >= RAIN_EVENT_THRESHOLD_MM
wind_values = test_df["wind_max_m_s"].to_numpy(dtype=float) if "wind_max_m_s" in test_df.columns else np.full(len(test_df), np.nan) wind_values = test_df["wind_max_m_s"].to_numpy(dtype=float) if "wind_max_m_s" in test_df.columns else np.full(len(test_df), np.nan)
@@ -545,8 +567,8 @@ def evaluate_sliced_performance(
("nighttime_utc", np.asarray(~is_day, dtype=bool), "18:00-05:59 UTC"), ("nighttime_utc", np.asarray(~is_day, dtype=bool), "18:00-05:59 UTC"),
("rainy_weeks", np.asarray(rainy_week, dtype=bool), "weeks with positive-rate >= test positive-rate"), ("rainy_weeks", np.asarray(rainy_week, dtype=bool), "weeks with positive-rate >= test positive-rate"),
("non_rainy_weeks", np.asarray(~rainy_week, dtype=bool), "weeks with positive-rate < test positive-rate"), ("non_rainy_weeks", np.asarray(~rainy_week, dtype=bool), "weeks with positive-rate < test positive-rate"),
("wet_context_last_1h", np.asarray(wet_context, dtype=bool), f"rain_last_1h_mm >= {RAIN_EVENT_THRESHOLD_MM:.2f}"), ("wet_context_recent_rain", np.asarray(wet_context, dtype=bool), f"{context_label} >= {RAIN_EVENT_THRESHOLD_MM:.2f}"),
("dry_context_last_1h", np.asarray(~wet_context, dtype=bool), f"rain_last_1h_mm < {RAIN_EVENT_THRESHOLD_MM:.2f}"), ("dry_context_recent_rain", np.asarray(~wet_context, dtype=bool), f"{context_label} < {RAIN_EVENT_THRESHOLD_MM:.2f}"),
("windy_q75", np.asarray(windy, dtype=bool), "wind_max_m_s >= test 75th percentile"), ("windy_q75", np.asarray(windy, dtype=bool), "wind_max_m_s >= test 75th percentile"),
("calm_below_q75", np.asarray(~windy, dtype=bool), "wind_max_m_s < test 75th percentile"), ("calm_below_q75", np.asarray(~windy, dtype=bool), "wind_max_m_s < test 75th percentile"),
] ]
@@ -585,6 +607,7 @@ def evaluate_sliced_performance(
def tune_threshold_walk_forward( def tune_threshold_walk_forward(
model_df, model_df,
feature_cols: list[str], feature_cols: list[str],
target_col: str,
model_family: str, model_family: str,
model_params: dict[str, Any], model_params: dict[str, Any],
calibration_method: str, calibration_method: str,
@@ -627,8 +650,8 @@ def tune_threshold_walk_forward(
if len(fold_train) < 160 or len(fold_test) < 25: if len(fold_train) < 160 or len(fold_test) < 25:
continue continue
y_fold_train = fold_train["rain_next_1h"].astype(int).to_numpy() y_fold_train = fold_train[target_col].astype(int).to_numpy()
y_fold_test = fold_test["rain_next_1h"].astype(int).to_numpy() y_fold_test = fold_test[target_col].astype(int).to_numpy()
if len(np.unique(y_fold_train)) < 2: if len(np.unique(y_fold_train)) < 2:
continue continue
@@ -706,6 +729,7 @@ def tune_threshold_walk_forward(
def walk_forward_backtest( def walk_forward_backtest(
model_df, model_df,
feature_cols: list[str], feature_cols: list[str],
target_col: str,
model_family: str, model_family: str,
model_params: dict[str, Any], model_params: dict[str, Any],
calibration_method: str, calibration_method: str,
@@ -745,8 +769,8 @@ def walk_forward_backtest(
if len(fold_train) < 160 or len(fold_test) < 25: if len(fold_train) < 160 or len(fold_test) < 25:
continue continue
y_fold_train = fold_train["rain_next_1h"].astype(int).to_numpy() y_fold_train = fold_train[target_col].astype(int).to_numpy()
y_fold_test = fold_test["rain_next_1h"].astype(int).to_numpy() y_fold_test = fold_test[target_col].astype(int).to_numpy()
if len(np.unique(y_fold_train)) < 2: if len(np.unique(y_fold_train)) < 2:
continue continue
@@ -755,8 +779,8 @@ def walk_forward_backtest(
continue continue
inner_train = fold_train.iloc[:-inner_val_rows] inner_train = fold_train.iloc[:-inner_val_rows]
inner_val = fold_train.iloc[-inner_val_rows:] inner_val = fold_train.iloc[-inner_val_rows:]
y_inner_train = inner_train["rain_next_1h"].astype(int).to_numpy() y_inner_train = inner_train[target_col].astype(int).to_numpy()
y_inner_val = inner_val["rain_next_1h"].astype(int).to_numpy() y_inner_val = inner_val[target_col].astype(int).to_numpy()
if len(np.unique(y_inner_train)) < 2: if len(np.unique(y_inner_train)) < 2:
continue continue
@@ -987,6 +1011,11 @@ def main() -> int:
start = parse_time(args.start) if args.start else "" start = parse_time(args.start) if args.start else ""
end = parse_time(args.end) if args.end else "" end = parse_time(args.end) if args.end else ""
horizon_hours = normalize_horizon_hours(args.horizon_hours)
horizon_label = horizon_suffix(horizon_hours)
target_col = rain_next_flag_col(horizon_hours)
target_mm_col = rain_next_mm_col(horizon_hours)
persistence_context_col = rain_last_mm_col(horizon_hours)
feature_cols = feature_columns_for_set(args.feature_set) feature_cols = feature_columns_for_set(args.feature_set)
needs_forecast = feature_columns_need_forecast(feature_cols) needs_forecast = feature_columns_need_forecast(feature_cols)
calibration_methods = parse_calibration_methods(args.calibration_methods) calibration_methods = parse_calibration_methods(args.calibration_methods)
@@ -1011,8 +1040,21 @@ def main() -> int:
return 0 return 0
raise RuntimeError(message) raise RuntimeError(message)
full_df = build_dataset(ws90, baro, forecast=forecast, rain_event_threshold_mm=RAIN_EVENT_THRESHOLD_MM) full_df = build_dataset(
model_df = model_frame(full_df, feature_cols, require_target=True) ws90,
baro,
forecast=forecast,
rain_event_threshold_mm=RAIN_EVENT_THRESHOLD_MM,
horizon_hours=horizon_hours,
)
if persistence_context_col not in full_df.columns:
persistence_context_col = rain_last_mm_col(1)
model_df = model_frame(
full_df,
feature_cols,
require_target=True,
target_col=target_col,
)
if len(model_df) < args.min_rows: if len(model_df) < args.min_rows:
message = f"not enough model-ready rows after filtering (need >= {args.min_rows})" message = f"not enough model-ready rows after filtering (need >= {args.min_rows})"
if args.allow_empty: if args.allow_empty:
@@ -1027,11 +1069,11 @@ def main() -> int:
) )
x_train = train_df[feature_cols] x_train = train_df[feature_cols]
y_train = train_df["rain_next_1h"].astype(int).to_numpy() y_train = train_df[target_col].astype(int).to_numpy()
x_val = val_df[feature_cols] x_val = val_df[feature_cols]
y_val = val_df["rain_next_1h"].astype(int).to_numpy() y_val = val_df[target_col].astype(int).to_numpy()
x_test = test_df[feature_cols] x_test = test_df[feature_cols]
y_test = test_df["rain_next_1h"].astype(int).to_numpy() y_test = test_df[target_col].astype(int).to_numpy()
if len(np.unique(y_train)) < 2: if len(np.unique(y_train)) < 2:
raise RuntimeError("training split does not contain both classes; cannot train classifier") raise RuntimeError("training split does not contain both classes; cannot train classifier")
@@ -1073,6 +1115,7 @@ def main() -> int:
threshold_tuning_walk_forward = tune_threshold_walk_forward( threshold_tuning_walk_forward = tune_threshold_walk_forward(
model_df=model_df.iloc[: len(train_df) + len(val_df)], model_df=model_df.iloc[: len(train_df) + len(val_df)],
feature_cols=feature_cols, feature_cols=feature_cols,
target_col=target_col,
model_family=selected_model_family, model_family=selected_model_family,
model_params=selected_model_params, model_params=selected_model_params,
calibration_method=selected_calibration_method, calibration_method=selected_calibration_method,
@@ -1093,7 +1136,7 @@ def main() -> int:
train_val_df = model_df.iloc[: len(train_df) + len(val_df)] train_val_df = model_df.iloc[: len(train_df) + len(val_df)]
x_train_val = train_val_df[feature_cols] x_train_val = train_val_df[feature_cols]
y_train_val = train_val_df["rain_next_1h"].astype(int).to_numpy() y_train_val = train_val_df[target_col].astype(int).to_numpy()
final_model, final_fit_info = fit_with_optional_calibration( final_model, final_fit_info = fit_with_optional_calibration(
model_family=selected_model_family, model_family=selected_model_family,
@@ -1109,16 +1152,23 @@ def main() -> int:
test_calibration = { test_calibration = {
"ece_10": expected_calibration_error(y_true=y_test, y_prob=y_test_prob, bins=10), "ece_10": expected_calibration_error(y_true=y_test, y_prob=y_test_prob, bins=10),
} }
naive_baselines_test = evaluate_naive_baselines(test_df=test_df, y_test=y_test) naive_baselines_test = evaluate_naive_baselines(
test_df=test_df,
y_test=y_test,
persistence_context_col=persistence_context_col,
)
sliced_performance = evaluate_sliced_performance( sliced_performance = evaluate_sliced_performance(
test_df=test_df, test_df=test_df,
y_true=y_test, y_true=y_test,
y_prob=y_test_prob, y_prob=y_test_prob,
threshold=chosen_threshold, threshold=chosen_threshold,
context_col=persistence_context_col,
context_label=persistence_context_col,
) )
walk_forward = walk_forward_backtest( walk_forward = walk_forward_backtest(
model_df=model_df, model_df=model_df,
feature_cols=feature_cols, feature_cols=feature_cols,
target_col=target_col,
model_family=selected_model_family, model_family=selected_model_family,
model_params=selected_model_params, model_params=selected_model_params,
calibration_method=selected_calibration_method, calibration_method=selected_calibration_method,
@@ -1135,8 +1185,12 @@ def main() -> int:
"model_family_requested": args.model_family, "model_family_requested": args.model_family,
"model_family": selected_model_family, "model_family": selected_model_family,
"model_params": selected_model_params, "model_params": selected_model_params,
"horizon_hours": horizon_hours,
"horizon_label": horizon_label,
"feature_set": args.feature_set, "feature_set": args.feature_set,
"target_definition": f"rain_next_1h_mm >= {RAIN_EVENT_THRESHOLD_MM:.2f}", "target_column": target_col,
"target_mm_column": target_mm_col,
"target_definition": f"{target_mm_col} >= {RAIN_EVENT_THRESHOLD_MM:.2f}",
"feature_columns": feature_cols, "feature_columns": feature_cols,
"forecast_model": args.forecast_model if needs_forecast else None, "forecast_model": args.forecast_model if needs_forecast else None,
"calibration_method_requested": calibration_methods, "calibration_method_requested": calibration_methods,
@@ -1207,6 +1261,7 @@ def main() -> int:
print(f" site: {args.site}") print(f" site: {args.site}")
print(f" model_version: {args.model_version}") print(f" model_version: {args.model_version}")
print(f" model_family: {selected_model_family} (requested={args.model_family})") print(f" model_family: {selected_model_family} (requested={args.model_family})")
print(f" horizon: {horizon_hours}h")
print(f" model_params: {selected_model_params}") print(f" model_params: {selected_model_params}")
print(f" calibration_method: {report['calibration_method']}") print(f" calibration_method: {report['calibration_method']}")
print( print(
@@ -1298,7 +1353,7 @@ def main() -> int:
dataset_dir = os.path.dirname(dataset_out) dataset_dir = os.path.dirname(dataset_out)
if dataset_dir: if dataset_dir:
os.makedirs(dataset_dir, exist_ok=True) os.makedirs(dataset_dir, exist_ok=True)
snapshot_cols = list(dict.fromkeys(feature_cols + ["rain_next_1h", "rain_next_1h_mm"])) snapshot_cols = list(dict.fromkeys(feature_cols + [target_col, target_mm_col]))
model_df[snapshot_cols].to_csv(dataset_out, index=True, index_label="ts") model_df[snapshot_cols].to_csv(dataset_out, index=True, index_label="ts")
print(f"Saved dataset snapshot to {dataset_out}") print(f"Saved dataset snapshot to {dataset_out}")
@@ -1320,6 +1375,10 @@ def main() -> int:
"forecast_model": args.forecast_model if needs_forecast else None, "forecast_model": args.forecast_model if needs_forecast else None,
"threshold": float(chosen_threshold), "threshold": float(chosen_threshold),
"target_mm": float(RAIN_EVENT_THRESHOLD_MM), "target_mm": float(RAIN_EVENT_THRESHOLD_MM),
"horizon_hours": horizon_hours,
"target_col": target_col,
"target_mm_col": target_mm_col,
"persistence_context_col": persistence_context_col,
"model_version": args.model_version, "model_version": args.model_version,
"trained_at": datetime.now(timezone.utc).isoformat(), "trained_at": datetime.now(timezone.utc).isoformat(),
"split": report["split"], "split": report["split"],
+22
View File
@@ -55,3 +55,25 @@ Priority key: `P0` = critical/blocking, `P1` = important, `P2` = later optimizat
- [x] [P0] Train baseline model on full available history and capture metrics. (completed on runtime machine) - [x] [P0] Train baseline model on full available history and capture metrics. (completed on runtime machine)
- [x] [P1] Add one expanded feature set and rerun evaluation. (completed on runtime machine 2026-03-12 with `feature_set=extended`, `model_version=rain-auto-v1-extended-202603120932`) - [x] [P1] Add one expanded feature set and rerun evaluation. (completed on runtime machine 2026-03-12 with `feature_set=extended`, `model_version=rain-auto-v1-extended-202603120932`)
- [x] [P0] Decide v1 threshold and define deployment interface. - [x] [P0] Decide v1 threshold and define deployment interface.
## 9) Extension Plan: 4-Hour Precipitation Window (Not Started)
- [x] [P0] Lock v2 target definition for horizon extension: `rain_next_4h_mm >= <threshold_mm>` and explicitly decide whether the threshold remains `0.2mm` or is increased for 4-hour labeling. (implemented with `0.2mm` carry-forward)
- [x] [P0] Decide rollout strategy: additive dual-horizon support (`1h` + `4h`) vs direct replacement; prefer dual-horizon for safe cutover. (implemented as additive dual-horizon)
- [x] [P0] Parameterize label horizon in shared ML code (`scripts/rain_model_common.py`) so target columns are generated for 4-hour windows (48 x 5-minute buckets) instead of hard-coded 1-hour columns.
- [x] [P1] Revisit persistence/context features currently tied to `rain_last_1h_mm`; decide whether to keep 1-hour context, add 4-hour context, or both for the 4-hour target. (implemented horizon-aware context column selection)
- [x] [P0] Update training pipeline (`scripts/train_rain_model.py`) to train against the 4-hour target column, including reports, model-card content, dataset snapshot columns, and artifact metadata.
- [x] [P0] Update audit pipeline (`scripts/audit_rain_data.py`) to report class balance and target definition for 4-hour labels.
- [x] [P0] Update inference pipeline (`scripts/predict_rain_model.py`) to use the 4-hour target, including realized-outcome availability checks (`pred_ts + 4h`) and metadata/reporting fields.
- [x] [P0] Finalize DB storage design for 4-hour predictions (new `predictions_rain_4h` table vs generic horizon column strategy) before migrations. (implemented dedicated `predictions_rain_4h` table)
- [x] [P0] Create schema migration (recommended: new hypertable `predictions_rain_4h` with `rain_next_4h_mm_actual` and `rain_next_4h_actual` fields) and matching indexes.
- [x] [P0] Update prediction upsert SQL to write to the 4-hour prediction table/columns.
- [x] [P0] Update monitoring views in `db/init/002_rain_monitoring_views.sql` so drift/calibration/pipeline-health views include the 4-hour prediction path.
- [x] [P0] Update Go DB query layer (`internal/db/series.go`) to read 4-hour prediction rows/fields.
- [x] [P1] Update dashboard API defaults (`cmd/ingestd/web.go`) from `rain_next_1h` to the selected 4-hour model name (or make model name configurable).
- [x] [P1] Update web UI labels/semantics (`cmd/ingestd/web/index.html`, `cmd/ingestd/web/app.js`) from “Rain 1h %” to “Rain 4h %” and verify chart legends/tooltips match the new horizon.
- [x] [P1] Update worker/runtime defaults (`docker-compose.yml`, `scripts/run_rain_ml_worker.py`, `scripts/run_p0_rain_workflow.sh`) to use `rain_next_4h` naming/versioning.
- [x] [P0] Update health-check defaults (`scripts/check_rain_pipeline_health.py`) for 4-hour evaluation latency (e.g., pending-eval age threshold > 4h).
- [x] [P1] Update docs and runbooks (`README.md`, `docs/rain_prediction.md`, `docs/rain_model_runbook.md`) so commands, table names, and target definitions match the 4-hour system.
- [ ] [P0] Run full retraining/evaluation for the 4-hour target and compare against current 1-hour model metrics before production cutover.
- [ ] [P0] Execute staged rollout: deploy schema + views, deploy model + inference, verify dashboard/health checks, then switch default model name.
- [x] [P1] Keep rollback path documented: retain `rain_next_1h` artifacts/table access until 4-hour monitoring is stable. (documented in `docs/rain_model_runbook.md` staged rollout/rollback section)