feat(rust): ping-collector, phase 1 of the Rust rewrite
CI / Format & lint (push) Successful in 19s
CI / Test suite (push) Successful in 47s
Docs / Validate documentation (push) Successful in 19s
Rust / fmt, clippy, test, build (push) Successful in 7m35s
Security / Dependency vulnerabilities (push) Successful in 1m37s
Security / Static analysis (push) Successful in 17s
Security / License report (push) Successful in 1m30s
CI / Format & lint (push) Successful in 19s
CI / Test suite (push) Successful in 47s
Docs / Validate documentation (push) Successful in 19s
Rust / fmt, clippy, test, build (push) Successful in 7m35s
Security / Dependency vulnerabilities (push) Successful in 1m37s
Security / Static analysis (push) Successful in 17s
Security / License report (push) Successful in 1m30s
rust/collector replaces the collection half of the Python web app: RID hourly gauges (with the late-report semantics: store everything, retry every 10 min while RID is behind, re-fetch yesterday's holes, startup gap fill), HII rain + water level, RID dams (today + yesterday) and Open-Meteo rain. Same tables and upsert semantics, one UNNEST statement per table, Bangkok wall-clock time regardless of the server time zone. Python side: COLLECTION_ENABLED=false makes the API leader stop collecting and instead re-forecast + notify when the newest RID reading in the database advances (polled every FORECAST_POLL_SECONDS) and once per clock hour; /scraping/status then serves the collector's status file; /scrape/trigger answers 409. Verified: 20 Rust tests (parsers, gap/schedule logic, database tests against PostgreSQL); live parity run of both collectors into two empty databases: all 9 tables row-for-row identical (scripts/collector_parity.py); the Python API served health, measurements, stats, status and model forecasts from a database filled only by ping-collector. 177 Python tests pass. reqwest uses native-tls: app.rid.go.th offers only TLS 1.2 CBC suites, which rustls refuses. Deploy/cutover/rollback: scripts/install_collector.sh, scripts/ping-collector.service; CI: .gitea/workflows/rust.yml (fmt, clippy, tests with a PostgreSQL service, release artifact).
This commit is contained in:
1 parent
e3694d7591
commit
4e619aa2ae
26 files changed
+5836
-15
No files matched your search
@@ -0,0 +1,84 @@
|
||||
//! RID station mapping: report column number (1..16) -> station metadata.
|
||||
//! Same precedence as the Python scraper: the runtime file
|
||||
//! (STATION_CONFIG_PATH, edited through the station CRUD API) if it exists,
|
||||
//! else the defaults bundled in src/data/stations.json.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::Deserialize;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
const BUNDLED: &str = include_str!("../../../src/data/stations.json");
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct StationInfo {
|
||||
pub code: String,
|
||||
pub thai_name: String,
|
||||
pub english_name: String,
|
||||
pub latitude: Option<f64>,
|
||||
pub longitude: Option<f64>,
|
||||
pub geohash: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Stations(BTreeMap<u32, StationInfo>);
|
||||
|
||||
impl Stations {
|
||||
pub fn load(runtime_path: &Path) -> Result<Self> {
|
||||
if runtime_path.exists() {
|
||||
let text = std::fs::read_to_string(runtime_path)
|
||||
.with_context(|| format!("reading {}", runtime_path.display()))?;
|
||||
let stations = Self::parse(&text)
|
||||
.with_context(|| format!("parsing {}", runtime_path.display()))?;
|
||||
tracing::info!(
|
||||
"loaded {} stations from {}",
|
||||
stations.0.len(),
|
||||
runtime_path.display()
|
||||
);
|
||||
return Ok(stations);
|
||||
}
|
||||
let stations = Self::bundled();
|
||||
tracing::info!("loaded {} bundled default stations", stations.0.len());
|
||||
Ok(stations)
|
||||
}
|
||||
|
||||
pub fn bundled() -> Self {
|
||||
Self::parse(BUNDLED).expect("bundled stations.json is valid")
|
||||
}
|
||||
|
||||
fn parse(text: &str) -> Result<Self> {
|
||||
let raw: BTreeMap<String, StationInfo> = serde_json::from_str(text)?;
|
||||
let map = raw
|
||||
.into_iter()
|
||||
.map(|(k, v)| Ok((k.trim().parse::<u32>()?, v)))
|
||||
.collect::<Result<BTreeMap<_, _>>>()?;
|
||||
Ok(Stations(map))
|
||||
}
|
||||
|
||||
/// Metadata for report column `n`, with the Python fallback for unknown
|
||||
/// columns (`P.{19+n}` / `Station n`).
|
||||
pub fn get(&self, n: u32) -> StationInfo {
|
||||
self.0.get(&n).cloned().unwrap_or_else(|| StationInfo {
|
||||
code: format!("P.{}", 19 + n),
|
||||
thai_name: format!("Station {n}"),
|
||||
english_name: format!("Station {n}"),
|
||||
latitude: None,
|
||||
longitude: None,
|
||||
geohash: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn bundled_mapping_has_the_16_rid_gauges() {
|
||||
let s = Stations::bundled();
|
||||
assert_eq!(s.0.len(), 16);
|
||||
assert_eq!(s.get(1).code, "P.20");
|
||||
assert_eq!(s.get(8).code, "P.1");
|
||||
assert_eq!(s.get(99).code, "P.118"); // Python fallback
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user