chore(rust): gap-fill progress lines; slow-statement warning at 30 s

rust/Cargo.lock

@@ -1163,6 +1163,7 @@ dependencies = [
  "anyhow",
  "chrono",
  "clap",
+ "log",
  "reqwest",
  "serde",
  "serde_json",

rust/Cargo.toml

@@ -15,6 +15,7 @@ repository = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor"
 anyhow = "1"
 chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] }
 clap = { version = "4", features = ["derive"] }
+log = "0.4"
 # native-tls (OpenSSL on Linux, SChannel on Windows), not rustls: app.rid.go.th (dams)
 # offers only TLS 1.2 CBC suites (ECDHE-RSA-AES256-SHA384), which rustls refuses.
 reqwest = { version = "0.12", default-features = false, features = ["native-tls", "json"] }

rust/collector/Cargo.toml

@@ -12,6 +12,7 @@ repository.workspace = true
 anyhow.workspace = true
 chrono.workspace = true
 clap.workspace = true
+log.workspace = true
 reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true

rust/collector/src/db.rs

@@ -15,10 +15,17 @@ use sqlx::postgres::PgPoolOptions;
 use std::collections::{BTreeMap, HashMap, HashSet};
 
 pub async fn connect(url: &str) -> Result<PgPool> {
+    use sqlx::ConnectOptions;
+    use std::str::FromStr;
+    // sqlx warns about any statement over 1 s; the whole-history gap scan
+    // legitimately takes several seconds on ~1M rows.
+    let options = sqlx::postgres::PgConnectOptions::from_str(url)
+        .context("invalid database URL")?
+        .log_slow_statements(log::LevelFilter::Warn, std::time::Duration::from_secs(30));
     PgPoolOptions::new()
         .max_connections(4)
         .acquire_timeout(std::time::Duration::from_secs(30))
-        .connect(url)
+        .connect_with(options)
         .await
         .context("connecting to PostgreSQL")
 }

rust/collector/src/rid.rs

@@ -301,6 +301,7 @@ pub async fn fill_gaps(
         dates.len()
     );
     let mut filled = 0;
+    let mut empty = 0;
     for (i, d) in dates.iter().enumerate() {
         if i > 0 {
             tokio::time::sleep(std::time::Duration::from_secs(1)).await; // be polite
@@ -310,11 +311,24 @@ pub async fn fill_gaps(
                 db::save_rid(pool, &readings, stations).await?;
                 filled += readings.len();
             }
-            Ok(_) => tracing::info!("gap fill {d}: RID has no data"),
+            Ok(_) => {
+                empty += 1;
+                tracing::debug!("gap fill {d}: RID has no data");
+            }
             Err(e) => tracing::warn!("gap fill {d}: {e:#}"),
         }
+        if (i + 1) % 50 == 0 && i + 1 < dates.len() {
+            tracing::info!(
+                "gap fill: {}/{} reports, at {d}, {filled} readings re-saved",
+                i + 1,
+                dates.len()
+            );
         }
-    tracing::info!("gap fill done: {filled} readings re-saved");
+    }
+    tracing::info!(
+        "gap fill done: {filled} readings re-saved from {} report(s); {empty} had no data at RID",
+        dates.len() - empty
+    );
     Ok(filled)
 }