From 536ec7ad70773033f75f7e450135fa11e52e8832 Mon Sep 17 00:00:00 2001 From: Nathan Coad Date: Wed, 28 Jan 2026 17:00:24 +1100 Subject: [PATCH] improve charts and logs --- cmd/ingestd/web/app.js | 11 ----------- cmd/ingestd/web/index.html | 9 --------- internal/db/series.go | 13 +++++++++---- internal/providers/openmeteo.go | 21 ++++++++++++++++----- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/cmd/ingestd/web/app.js b/cmd/ingestd/web/app.js index 553eab1..157da00 100644 --- a/cmd/ingestd/web/app.js +++ b/cmd/ingestd/web/app.js @@ -429,7 +429,6 @@ function renderDashboard(data) { const fcWinds = forecast.map((p) => p.wind_m_s); const fcGusts = forecast.map((p) => p.wind_gust_m_s); const fcRH = forecast.map((p) => p.rh); - const fcPrecip = forecast.map((p) => p.precip_mm); const obsTempSummary = minMax(obsTemps); const obsWindSummary = minMax(obsWinds); @@ -580,16 +579,6 @@ function renderDashboard(data) { }; upsertChart("chart-rain", rainChart); - const precipChart = { - type: "bar", - data: { - datasets: [ - { label: "forecast precip mm", data: series(forecast, "precip_mm"), backgroundColor: colors.precip }, - ], - }, - options: baseOptions(range), - }; - upsertChart("chart-precip", precipChart); updateSingleChartMode(); } diff --git a/cmd/ingestd/web/index.html b/cmd/ingestd/web/index.html index 184d9c9..5bdbfdf 100644 --- a/cmd/ingestd/web/index.html +++ b/cmd/ingestd/web/index.html @@ -155,15 +155,6 @@ -
-
-
Precipitation (forecast)
- -
-
- -
-
diff --git a/internal/db/series.go b/internal/db/series.go index a4057ac..10d0aa1 100644 --- a/internal/db/series.go +++ b/internal/db/series.go @@ -34,6 +34,7 @@ type ForecastPoint struct { WindGustMS *float64 `json:"wind_gust_m_s,omitempty"` WindDirDeg *float64 `json:"wind_dir_deg,omitempty"` PrecipMM *float64 `json:"precip_mm,omitempty"` + PrecipProb *float64 `json:"precip_prob,omitempty"` CloudCover *float64 `json:"cloud_cover,omitempty"` } @@ -195,6 +196,7 @@ func (d *DB) ForecastSeriesLatest(ctx context.Context, site, model string) (Fore wind_gust_m_s, wind_dir_deg, precip_mm, + precip_prob, cloud_cover FROM forecast_openmeteo_hourly WHERE site = $1 AND model = $2 AND retrieved_at = $3 @@ -210,9 +212,9 @@ func (d *DB) ForecastSeriesLatest(ctx context.Context, site, model string) (Fore var ( ts time.Time temp, rh, msl, wind, gust sql.NullFloat64 - dir, precip, cloud sql.NullFloat64 + dir, precip, prob, cloud sql.NullFloat64 ) - if err := rows.Scan(&ts, &temp, &rh, &msl, &wind, &gust, &dir, &precip, &cloud); err != nil { + if err := rows.Scan(&ts, &temp, &rh, &msl, &wind, &gust, &dir, &precip, &prob, &cloud); err != nil { return ForecastSeries{}, err } points = append(points, ForecastPoint{ @@ -224,6 +226,7 @@ func (d *DB) ForecastSeriesLatest(ctx context.Context, site, model string) (Fore WindGustMS: nullFloatPtr(gust), WindDirDeg: nullFloatPtr(dir), PrecipMM: nullFloatPtr(precip), + PrecipProb: nullFloatPtr(prob), CloudCover: nullFloatPtr(cloud), }) } @@ -249,6 +252,7 @@ func (d *DB) ForecastSeriesRange(ctx context.Context, site, model string, start, wind_gust_m_s, wind_dir_deg, precip_mm, + precip_prob, cloud_cover FROM forecast_openmeteo_hourly WHERE site = $1 AND model = $2 AND ts >= $3 AND ts <= $4 @@ -266,9 +270,9 @@ func (d *DB) ForecastSeriesRange(ctx context.Context, site, model string, start, ts time.Time retrieved time.Time temp, rh, msl, wind, gust sql.NullFloat64 - dir, precip, cloud sql.NullFloat64 + dir, precip, prob, cloud sql.NullFloat64 ) - if err := rows.Scan(&ts, &retrieved, &temp, &rh, &msl, &wind, &gust, &dir, &precip, &cloud); err != nil { + if err := rows.Scan(&ts, &retrieved, &temp, &rh, &msl, &wind, &gust, &dir, &precip, &prob, &cloud); err != nil { return ForecastSeries{}, err } if retrieved.After(maxRetrieved) { @@ -283,6 +287,7 @@ func (d *DB) ForecastSeriesRange(ctx context.Context, site, model string, start, WindGustMS: nullFloatPtr(gust), WindDirDeg: nullFloatPtr(dir), PrecipMM: nullFloatPtr(precip), + PrecipProb: nullFloatPtr(prob), CloudCover: nullFloatPtr(cloud), }) } diff --git a/internal/providers/openmeteo.go b/internal/providers/openmeteo.go index b290b58..36911f9 100644 --- a/internal/providers/openmeteo.go +++ b/internal/providers/openmeteo.go @@ -77,23 +77,34 @@ func (o *OpenMeteo) Fetch(ctxDone <-chan struct{}, site Site, model string) (*Fo } defer resp.Body.Close() + const maxLogBody = int64(65536) + bodyBuf, _ := io.ReadAll(io.LimitReader(resp.Body, maxLogBody+1)) + truncated := int64(len(bodyBuf)) > maxLogBody + if truncated { + bodyBuf = bodyBuf[:maxLogBody] + } + if truncated { + log.Printf("open-meteo response status=%d bytes=%d truncated=true body=%s", resp.StatusCode, len(bodyBuf), string(bodyBuf)) + } else { + log.Printf("open-meteo response status=%d bytes=%d body=%s", resp.StatusCode, len(bodyBuf), string(bodyBuf)) + } + if resp.StatusCode/100 != 2 { - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) var apiErr struct { Error bool `json:"error"` Reason string `json:"reason"` } - if err := json.Unmarshal(body, &apiErr); err == nil && apiErr.Reason != "" { + if err := json.Unmarshal(bodyBuf, &apiErr); err == nil && apiErr.Reason != "" { return nil, fmt.Errorf("open-meteo HTTP %d: %s", resp.StatusCode, apiErr.Reason) } - if len(body) > 0 { - return nil, fmt.Errorf("open-meteo HTTP %d: %s", resp.StatusCode, string(body)) + if len(bodyBuf) > 0 { + return nil, fmt.Errorf("open-meteo HTTP %d: %s", resp.StatusCode, string(bodyBuf)) } return nil, fmt.Errorf("open-meteo HTTP %d", resp.StatusCode) } var raw map[string]any - if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil { + if err := json.Unmarshal(bodyBuf, &raw); err != nil { return nil, err }