security: pip-audit + bandit gates that can fail; patch 29 known CVEs

security.yml previously ran safety/bandit/semgrep with `|| true` and could not go red. Now: pip-audit on requirements.txt is a hard gate (dev deps reported only), bandit HIGH fails (B104 bind-all skipped: intended behind Cloudflare/Caddy), pip-licenses uploaded as a report. Weekly + on dependency/source changes. Running it locally found 29 advisories, all in pinned-and-forgotten runtime deps: starlette 0.27 (7, incl. Host-header path confusion and form DoS), fastapi 0.104, requests 2.31 (3), pymysql 1.1. Bumped to current: fastapi 0.141.1 / starlette 1.6.0, pydantic 2.13.5, uvicorn 0.52.4, requests 2.34.2, pymysql 1.2.0; dev: pytest 9.1.1, black 26.5.1. pip-audit is now clean. requires-python narrowed to 3.11 (the truth: psycopg2-binary 2.9.9 fails on 3.13; pandas 2.0.3 has no 3.12 wheels). Full suite passes; API smoke-tested (health, stations, forecast, history, stats, docs, openapi) on the new stack. black 26 reformatted 8 files.
2026-09-11 23:44:43 +02:00
parent 97a6694ab2
commit b03318210c
14 changed files with 285 additions and 1048 deletions
@@ -38,7 +38,7 @@ jobs:
      - name: Install tools
        run: |
          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore black==23.11.0 isort==5.12.0 flake8==6.1.0
+          pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0

      - name: black
        run: black --check --diff src/ *.py
@@ -67,7 +67,7 @@ jobs:
        run: |
          python -m pip install --upgrade pip --root-user-action=ignore
          pip install --root-user-action=ignore -r requirements.txt
-          pip install --root-user-action=ignore pytest==7.4.3 pytest-asyncio==0.21.1
+          pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1

      - name: pytest
        env:
@@ -1,293 +1,104 @@
-name: Security & Dependency Updates
+name: Security
+
+# Two gates that can actually fail, plus one report:
+#   - pip-audit against requirements.txt: any known vulnerability in a runtime
+#     dependency fails the job (dev-only tools are reported, not gated)
+#   - bandit on src/: HIGH severity findings fail; medium/low are listed.
+#     B104 (bind 0.0.0.0) is skipped: the service is meant to listen on all
+#     interfaces behind Cloudflare/Caddy.
+#   - pip-licenses report as an artifact (informational; the project is MIT
+#     and its runtime deps are MIT/BSD/Apache/PSF)
+# The old file ran safety/bandit/semgrep with `|| true` and could not go red.

 on:
  schedule:
-    # Run security scans daily at 3 AM UTC
-    - cron: "0 3 * * *"
+    - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC
  workflow_dispatch:
  push:
    paths:
      - "requirements*.txt"
-      - "Dockerfile"
+      - "pyproject.toml"
+      - "uv.lock"
+      - "src/**/*.py"
      - ".gitea/workflows/security.yml"
+  pull_request:
+    paths:
+      - "requirements*.txt"
+      - "pyproject.toml"
+      - "src/**/*.py"

 env:
  PYTHON_VERSION: "3.11"
-  # GitHub token for better rate limits and authentication
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}

 jobs:
-  # Dependency vulnerability scan
-  dependency-scan:
-    name: Dependency Security Scan
+  dependencies:
+    name: Dependency vulnerabilities
    runs-on: ubuntu-latest
-
    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
+      - uses: actions/checkout@v4

-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

-      - name: Install dependencies
+      - name: Install pip-audit
        run: |
          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore safety bandit semgrep
+          pip install --root-user-action=ignore pip-audit

-      - name: Run Safety check
-        run: |
-          safety check -r requirements.txt --json --output safety-report.json || true
-          safety check -r requirements-dev.txt --json --output safety-dev-report.json || true
+      - name: Runtime dependencies (gate)
+        run: pip-audit -r requirements.txt --strict --desc on

-      - name: Run Bandit security scan
-        run: |
-          bandit -r src/ -f json -o bandit-report.json || true
+      - name: Dev dependencies (report only)
+        run: pip-audit -r requirements-dev.txt --desc on || echo "::warning::dev-only dependency advisories above"

-      - name: Run Semgrep security scan
-        run: |
-          semgrep --config=auto src/ --json --output=semgrep-report.json || true
-
-      - name: Upload security reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: security-reports-${{ github.run_number }}
-          path: |
-            safety-report.json
-            safety-dev-report.json
-            bandit-report.json
-            semgrep-report.json
-
-      - name: Check for critical vulnerabilities
-        run: |
-          echo "Checking for critical vulnerabilities..."
-
-          # Check Safety results
-          if [ -f safety-report.json ]; then
-            critical_count=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null || echo "0")
-            if [ "$critical_count" -gt 0 ]; then
-              echo "Found $critical_count dependency vulnerabilities"
-              jq '.vulnerabilities[] | "- \(.package_name) \(.installed_version): \(.vulnerability_id)"' safety-report.json
-            else
-              echo "No dependency vulnerabilities found"
-            fi
-          fi
-
-          # Check Bandit results
-          if [ -f bandit-report.json ]; then
-            high_severity=$(jq '.results[] | select(.issue_severity == "HIGH") | length' bandit-report.json 2>/dev/null | wc -l)
-            if [ "$high_severity" -gt 0 ]; then
-              echo "Found $high_severity high-severity security issues"
-            else
-              echo "No high-severity security issues found"
-            fi
-          fi
-
-  # License compliance check
-  license-check:
-    name: License Compliance
+  code:
+    name: Static analysis
    runs-on: ubuntu-latest
-
    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
+      - uses: actions/checkout@v4

-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

-      - name: Install pip-licenses
+      - name: Install bandit
        run: |
          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore pip-licenses
-          pip install --root-user-action=ignore -r requirements.txt
+          pip install --root-user-action=ignore bandit

-      - name: Check licenses
+      - name: bandit (HIGH fails; medium/low listed)
        run: |
-          echo "Checking dependency licenses..."
+          bandit -r src/ -q --skip B104 -ll -ii || true
+          bandit -r src/ -q --skip B104 --severity-level high --confidence-level medium
+
+  licenses:
+    name: License report
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: requirements.txt
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip --root-user-action=ignore
+          pip install --root-user-action=ignore -r requirements.txt pip-licenses
+
+      - name: Report
+        run: |
+          pip-licenses --format=markdown --with-urls --output-file=licenses.md
          pip-licenses --format=json --output-file=licenses.json
-          pip-licenses --format=markdown --output-file=licenses.md
+          echo "Copyleft licenses among runtime deps (informational):"
+          pip-licenses --format=plain | grep -iE 'GPL|AGPL|LGPL' || echo "  none"

-          # Check for problematic licenses
-          problematic_licenses=("GPL" "AGPL" "LGPL")
-
-          for license in "${problematic_licenses[@]}"; do
-            if grep -i "$license" licenses.json; then
-              echo "Found potentially problematic license: $license"
-            fi
-          done
-
-          echo "License check completed"
-
-      - name: Upload license report
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3
        with:
-          name: license-report-${{ github.run_number }}
+          name: licenses-${{ github.run_number }}
          path: |
-            licenses.json
            licenses.md
-
-  # Dependency update check
-  dependency-update:
-    name: Check for Dependency Updates
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install pip-check-updates equivalent
-        run: |
-          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore pip-review
-
-      - name: Check for outdated packages
-        run: |
-          echo "Checking for outdated packages..."
-          pip install --root-user-action=ignore -r requirements.txt
-          pip list --outdated --format=json > outdated-packages.json || true
-
-          if [ -s outdated-packages.json ]; then
-            echo "Outdated packages found:"
-            cat outdated-packages.json | jq -r '.[] | "- \(.name): \(.version) -> \(.latest_version)"'
-          else
-            echo "All packages are up to date"
-          fi
-
-      - name: Upload dependency reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: dependency-reports-${{ github.run_number }}
-          path: |
-            outdated-packages.json
-
-  # Code quality metrics
-  code-quality:
-    name: Code Quality Metrics
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install quality tools
-        run: |
-          python -m pip install --upgrade pip --root-user-action=ignore
-          pip install --root-user-action=ignore radon xenon vulture
-          pip install --root-user-action=ignore -r requirements.txt
-
-      - name: Calculate code complexity
-        run: |
-          echo "Calculating code complexity..."
-          radon cc src/ --json > complexity-report.json
-          radon mi src/ --json > maintainability-report.json
-
-          echo "Complexity Summary:"
-          radon cc src/ --average
-
-          echo "Maintainability Summary:"
-          radon mi src/
-
-      - name: Find dead code
-        run: |
-          echo "Checking for dead code..."
-          vulture src/ --json > dead-code-report.json || true
-
-      - name: Check for code smells
-        run: |
-          echo "Checking for code smells..."
-          xenon --max-absolute B --max-modules A --max-average A src/ || true
-
-      - name: Upload quality reports
-        uses: actions/upload-artifact@v3
-        with:
-          name: code-quality-reports-${{ github.run_number }}
-          path: |
-            complexity-report.json
-            maintainability-report.json
-            dead-code-report.json
-
-  # Security summary
-  security-summary:
-    name: Security Summary
-    runs-on: ubuntu-latest
-    needs: [dependency-scan, license-check, code-quality]
-    if: always()
-
-    steps:
-      - name: Download all artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Generate security summary
-        run: |
-          echo "# Security Scan Summary" > security-summary.md
-          echo "" >> security-summary.md
-          echo "**Scan Date:** $(date -u)" >> security-summary.md
-          echo "**Repository:** ${{ github.repository }}" >> security-summary.md
-          echo "**Commit:** ${{ github.sha }}" >> security-summary.md
-          echo "" >> security-summary.md
-
-          echo "## Results" >> security-summary.md
-          echo "" >> security-summary.md
-
-          # Dependency scan results
-          if [ -f security-reports-*/safety-report.json ]; then
-            vuln_count=$(jq '.vulnerabilities | length' security-reports-*/safety-report.json 2>/dev/null || echo "0")
-            if [ "$vuln_count" -eq 0 ]; then
-              echo "- Dependency Scan: No vulnerabilities found" >> security-summary.md
-            else
-              echo "- Dependency Scan: $vuln_count vulnerabilities found" >> security-summary.md
-            fi
-          else
-            echo "- Dependency Scan: Results not available" >> security-summary.md
-          fi
-
-          # Docker scan results (removed Trivy)
-          echo "- Docker Scan: Skipped (Trivy removed)" >> security-summary.md
-
-          # License check results
-          if [ -f license-report-*/licenses.json ]; then
-            echo "- License Check: Completed" >> security-summary.md
-          else
-            echo "- License Check: Results not available" >> security-summary.md
-          fi
-
-          # Code quality results
-          if [ -f code-quality-reports-*/complexity-report.json ]; then
-            echo "- Code Quality: Analyzed" >> security-summary.md
-          else
-            echo "- Code Quality: Results not available" >> security-summary.md
-          fi
-
-          echo "" >> security-summary.md
-          echo "## Detailed Reports" >> security-summary.md
-          echo "" >> security-summary.md
-          echo "Detailed reports are available in the workflow artifacts." >> security-summary.md
-
-          cat security-summary.md
-
-      - name: Upload security summary
-        uses: actions/upload-artifact@v3
-        with:
-          name: security-summary-${{ github.run_number }}
-          path: security-summary.md
+            licenses.json
@@ -19,7 +19,7 @@ repos:

  # Python code formatting with Black
  - repo: https://github.com/psf/black
-    rev: 23.11.0
+    rev: 26.5.1
    hooks:
      - id: black
        language_version: python3
@@ -34,23 +34,23 @@ classifiers = [
    "Environment :: Web Environment",
    "Framework :: FastAPI"
 ]
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.12"
 dependencies = [
    # Core dependencies
-    "requests==2.31.0",
+    "requests==2.34.2",
    "schedule==1.2.0",
    "pandas==2.0.3",
    "numpy>=1.24,<2",
    # Flood forecasting (ML)
    "scikit-learn==1.9.0",
    # Web API framework
-    "fastapi==0.104.1",
-    "uvicorn[standard]==0.24.0",
-    "pydantic==2.5.0",
+    "fastapi==0.141.1",
+    "uvicorn[standard]==0.52.4",
+    "pydantic==2.13.5",
    # Database adapters
    "sqlalchemy==2.0.23",
    "influxdb==5.3.1",
-    "pymysql==1.1.0",
+    "pymysql==1.2.0",
    "psycopg2-binary==2.9.9",
    # Monitoring and metrics
    "psutil==5.9.6"
@@ -59,11 +59,11 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
    # Testing
-    "pytest==7.4.3",
+    "pytest==9.1.1",
    "pytest-cov==4.1.0",
    "pytest-asyncio==0.21.1",
    # Code formatting and linting
-    "black==23.11.0",
+    "black==26.5.1",
    "flake8==6.1.0",
    "isort==5.12.0",
    "mypy==1.7.1",
@@ -73,7 +73,7 @@ dev = [
    "ipython==8.17.2",
    "jupyter==1.0.0",
    # Type stubs
-    "types-requests==2.31.0.10",
+    "types-requests==2.33.0.20260906",
    "types-python-dateutil==2.8.19.14"
 ]
 docs = [
@@ -83,7 +83,7 @@ docs = [
 ]
 all = [
    "influxdb==5.3.1",
-    "pymysql==1.1.0",
+    "pymysql==1.2.0",
    "psycopg2-binary==2.9.9"
 ]

@@ -100,11 +100,11 @@ Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/
 [dependency-groups]
 dev = [
    # Testing
-    "pytest==7.4.3",
+    "pytest==9.1.1",
    "pytest-cov==4.1.0",
    "pytest-asyncio==0.21.1",
    # Code formatting and linting
-    "black==23.11.0",
+    "black==26.5.1",
    "flake8==6.1.0",
    "isort==5.12.0",
    "mypy==1.7.1",
@@ -114,7 +114,7 @@ dev = [
    "ipython==8.17.2",
    "jupyter==1.0.0",
    # Type stubs
-    "types-requests==2.31.0.10",
+    "types-requests==2.33.0.20260906",
    "types-python-dateutil==2.8.19.14",
    # Documentation
    "sphinx==7.2.6",
@@ -2,12 +2,12 @@
 -r requirements.txt

 # Testing
-pytest==7.4.3
+pytest==9.1.1
 pytest-cov==4.1.0
 pytest-asyncio==0.21.1

 # Code formatting and linting
-black==23.11.0
+black==26.5.1
 flake8==6.1.0
 isort==5.12.0
 mypy==1.7.1
@@ -25,5 +25,5 @@ ipython==8.17.2
 jupyter==1.0.0

 # Type stubs
-types-requests==2.31.0.10
+types-requests==2.33.0.20260906
 types-python-dateutil==2.8.19.14
@@ -1,5 +1,5 @@
 # Core dependencies
-requests==2.31.0
+requests==2.34.2
 schedule==1.2.0
 pandas==2.0.3
 numpy>=1.24,<2  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
@@ -8,23 +8,23 @@ numpy>=1.24,<2  # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
 scikit-learn==1.9.0

 # Web API framework
-fastapi==0.104.1
-uvicorn[standard]==0.24.0
-pydantic==2.5.0
+fastapi==0.141.1
+uvicorn[standard]==0.52.4
+pydantic==2.13.5

 # Database adapters
 sqlalchemy==2.0.23
 influxdb==5.3.1
-pymysql==1.1.0
+pymysql==1.2.0
 psycopg2-binary==2.9.9

 # Monitoring and metrics
 psutil==5.9.6

 # Development dependencies (optional)
-pytest==7.4.3
+pytest==9.1.1
 pytest-cov==4.1.0
-black==23.11.0
+black==26.5.1
 flake8==6.1.0
 mypy==1.7.1
 pre-commit==3.5.0
@@ -139,12 +139,16 @@ class InfluxDBAdapter(DatabaseAdapter):
                    "time": measurement["timestamp"].isoformat(),
                    "fields": {
                        "water_level": float(measurement["water_level"]),
-                        "discharge": float(measurement["discharge"])
+                        "discharge": (
+                            float(measurement["discharge"])
                            if measurement.get("discharge") is not None
-                        else None,
-                        "discharge_percent": float(measurement["discharge_percent"])
+                            else None
+                        ),
+                        "discharge_percent": (
+                            float(measurement["discharge_percent"])
                            if measurement.get("discharge_percent")
-                        else None,
+                            else None
+                        ),
                    },
                }
                points.append(point)
@@ -551,13 +555,13 @@ class SQLAdapter(DatabaseAdapter):
                            "station_code": row[1],
                            "station_name_en": row[2],
                            "station_name_th": row[3],
-                            "water_level": float(row[4])
-                            if row[4] is not None
-                            else None,
+                            "water_level": (
+                                float(row[4]) if row[4] is not None else None
+                            ),
                            "discharge": float(row[5]) if row[5] is not None else None,
-                            "discharge_percent": float(row[6])
-                            if row[6] is not None
-                            else None,
+                            "discharge_percent": (
+                                float(row[6]) if row[6] is not None else None
+                            ),
                            "status": row[7],
                        }
                    )
@@ -611,13 +615,13 @@ class SQLAdapter(DatabaseAdapter):
                            "station_code": row[1],
                            "station_name_en": row[2],
                            "station_name_th": row[3],
-                            "water_level": float(row[4])
-                            if row[4] is not None
-                            else None,
+                            "water_level": (
+                                float(row[4]) if row[4] is not None else None
+                            ),
                            "discharge": float(row[5]) if row[5] is not None else None,
-                            "discharge_percent": float(row[6])
-                            if row[6] is not None
-                            else None,
+                            "discharge_percent": (
+                                float(row[6]) if row[6] is not None else None
+                            ),
                            "status": row[7],
                        }
                    )
@@ -666,13 +670,13 @@ class SQLAdapter(DatabaseAdapter):
                            "station_id": row[1],
                            "station_code": row[2] or f"Station_{row[1]}",
                            "station_name_th": row[3] or f"Station {row[1]}",
-                            "water_level": float(row[4])
-                            if row[4] is not None
-                            else None,
+                            "water_level": (
+                                float(row[4]) if row[4] is not None else None
+                            ),
                            "discharge": float(row[5]) if row[5] is not None else None,
-                            "discharge_percent": float(row[6])
-                            if row[6] is not None
-                            else None,
+                            "discharge_percent": (
+                                float(row[6]) if row[6] is not None else None
+                            ),
                            "status": row[7],
                        }
                    )
@@ -125,9 +125,9 @@ class DatabaseHealthCheck(HealthCheck):
                "message": "Database connection OK",
                "details": {
                    "latest_data_count": len(latest_data),
-                    "latest_timestamp": str(latest_data[0].get("timestamp"))
-                    if latest_data
-                    else None,
+                    "latest_timestamp": (
+                        str(latest_data[0].get("timestamp")) if latest_data else None
+                    ),
                },
            }

@@ -207,9 +207,9 @@ def fill_from_hii(
                "timestamp": missing["timestamp"],
                "station_code": code,
                "water_level": missing["wl_msl"] - offset,
-                "discharge": missing["discharge"]
-                if code in _HII_EXACT_MIRRORS
-                else float("nan"),
+                "discharge": (
+                    missing["discharge"] if code in _HII_EXACT_MIRRORS else float("nan")
+                ),
            }
        )
        fills.append(fill)
@@ -320,9 +320,9 @@ def train_station(
                    skipped_heads,
                )
            else:
-                skipped_heads[
-                    head_key
-                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                skipped_heads[head_key] = (
+                    f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                )
            heads[head_key] = clf

            if not skip_eval:
@@ -417,9 +417,9 @@ def train_station(
                if clf is not None:
                    skipped_heads.pop(head_key, None)
            else:
-                skipped_heads[
-                    head_key
-                ] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                skipped_heads[head_key] = (
+                    f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
+                )
                final_heads[head_key] = None

    # v4 = + Mae Ngat dam features; v3 = rise + rain; v2 = rise target only
@@ -51,8 +51,7 @@ class PostgresHistory:
        if start >= end:
            raise ValueError("start must be before end")

-        query = text(
-            """
+        query = text("""
            SELECT m.timestamp, s.station_code, m.water_level,
                   m.discharge, m.discharge_percent
            FROM water_measurements m
@@ -62,8 +61,7 @@ class PostgresHistory:
              AND m.timestamp <= :end_time
            ORDER BY m.timestamp ASC
            LIMIT :limit
-            """
-        )
+            """)
        with self.engine.connect() as connection:
            rows = connection.execute(
                query,
@@ -91,9 +89,9 @@ class PostgresHistory:
                        "station_code": station_code,
                        "water_level": water_level,
                        "discharge": discharge,
-                        "discharge_percent": float(row[4])
-                        if row[4] is not None
-                        else None,
+                        "discharge_percent": (
+                            float(row[4]) if row[4] is not None else None
+                        ),
                    }
                )
            return result
@@ -173,8 +173,10 @@ class RequestTracker:
                "failed_requests": self.failed_requests,
                "success_rate": self.successful_requests / self.total_requests,
                "average_response_time": self.total_response_time / self.total_requests,
-                "last_request_time": self.last_request_time.isoformat()
+                "last_request_time": (
+                    self.last_request_time.isoformat()
                    if self.last_request_time
-                else None,
+                    else None
+                ),
                "error_breakdown": dict(self.error_count_by_type),
            }
@@ -323,9 +323,11 @@ class RidReservoirStore:
            if not preserve_cols:
                return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
            updates = ", ".join(
+                (
                    f"{c} = COALESCE(excluded.{c}, {table}.{c})"
                    if c in preserve_cols
                    else f"{c} = excluded.{c}"
+                )
                for c in value_cols
            )
            return (
@@ -334,9 +336,11 @@ class RidReservoirStore:
            )
        if self.db_type == "postgresql":
            updates = ", ".join(
+                (
                    f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
                    if c in preserve_cols
                    else f"{c} = EXCLUDED.{c}"
+                )
                for c in value_cols
            )
            return (
@@ -344,9 +348,11 @@ class RidReservoirStore:
                f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
            )
        updates = ", ".join(
+            (
                f"{c} = COALESCE(VALUES({c}), {c})"
                if c in preserve_cols
                else f"{c} = VALUES({c})"
+            )
            for c in value_cols
        )
        return (