security: pip-audit + bandit gates that can fail; patch 29 known CVEs

security.yml previously ran safety/bandit/semgrep with `|| true` and could
not go red. Now: pip-audit on requirements.txt is a hard gate (dev deps
reported only), bandit HIGH fails (B104 bind-all skipped: intended behind
Cloudflare/Caddy), pip-licenses uploaded as a report. Weekly + on
dependency/source changes.

Running it locally found 29 advisories, all in pinned-and-forgotten
runtime deps: starlette 0.27 (7, incl. Host-header path confusion and
form DoS), fastapi 0.104, requests 2.31 (3), pymysql 1.1. Bumped to
current: fastapi 0.141.1 / starlette 1.6.0, pydantic 2.13.5, uvicorn
0.52.4, requests 2.34.2, pymysql 1.2.0; dev: pytest 9.1.1, black 26.5.1.
pip-audit is now clean. requires-python narrowed to 3.11 (the truth:
psycopg2-binary 2.9.9 fails on 3.13; pandas 2.0.3 has no 3.12 wheels).
Full suite passes; API smoke-tested (health, stations, forecast, history,
stats, docs, openapi) on the new stack. black 26 reformatted 8 files.
This commit is contained in:
2026-09-11 23:44:43 +02:00
parent 97a6694ab2
commit b03318210c
14 changed files with 285 additions and 1048 deletions
+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
- name: Install tools - name: Install tools
run: | run: |
python -m pip install --upgrade pip --root-user-action=ignore python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore black==23.11.0 isort==5.12.0 flake8==6.1.0 pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0
- name: black - name: black
run: black --check --diff src/ *.py run: black --check --diff src/ *.py
@@ -67,7 +67,7 @@ jobs:
run: | run: |
python -m pip install --upgrade pip --root-user-action=ignore python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore -r requirements.txt pip install --root-user-action=ignore -r requirements.txt
pip install --root-user-action=ignore pytest==7.4.3 pytest-asyncio==0.21.1 pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1
- name: pytest - name: pytest
env: env:
+65 -254
View File
@@ -1,293 +1,104 @@
name: Security & Dependency Updates name: Security
# Two gates that can actually fail, plus one report:
# - pip-audit against requirements.txt: any known vulnerability in a runtime
# dependency fails the job (dev-only tools are reported, not gated)
# - bandit on src/: HIGH severity findings fail; medium/low are listed.
# B104 (bind 0.0.0.0) is skipped: the service is meant to listen on all
# interfaces behind Cloudflare/Caddy.
# - pip-licenses report as an artifact (informational; the project is MIT
# and its runtime deps are MIT/BSD/Apache/PSF)
# The old file ran safety/bandit/semgrep with `|| true` and could not go red.
on: on:
schedule: schedule:
# Run security scans daily at 3 AM UTC - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC
- cron: "0 3 * * *"
workflow_dispatch: workflow_dispatch:
push: push:
paths: paths:
- "requirements*.txt" - "requirements*.txt"
- "Dockerfile" - "pyproject.toml"
- "uv.lock"
- "src/**/*.py"
- ".gitea/workflows/security.yml" - ".gitea/workflows/security.yml"
pull_request:
paths:
- "requirements*.txt"
- "pyproject.toml"
- "src/**/*.py"
env: env:
PYTHON_VERSION: "3.11" PYTHON_VERSION: "3.11"
# GitHub token for better rate limits and authentication
GH_TOKEN: ${{ secrets.GH_TOKEN }}
jobs: jobs:
# Dependency vulnerability scan dependencies:
dependency-scan: name: Dependency vulnerabilities
name: Dependency Security Scan
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout code - uses: actions/checkout@v4
uses: actions/checkout@v4
with:
token: ${{ secrets.GITEA_TOKEN }}
- name: Set up Python - uses: actions/setup-python@v5
uses: actions/setup-python@v4
with: with:
python-version: ${{ env.PYTHON_VERSION }} python-version: ${{ env.PYTHON_VERSION }}
- name: Install dependencies - name: Install pip-audit
run: | run: |
python -m pip install --upgrade pip --root-user-action=ignore python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore safety bandit semgrep pip install --root-user-action=ignore pip-audit
- name: Run Safety check - name: Runtime dependencies (gate)
run: | run: pip-audit -r requirements.txt --strict --desc on
safety check -r requirements.txt --json --output safety-report.json || true
safety check -r requirements-dev.txt --json --output safety-dev-report.json || true
- name: Run Bandit security scan - name: Dev dependencies (report only)
run: | run: pip-audit -r requirements-dev.txt --desc on || echo "::warning::dev-only dependency advisories above"
bandit -r src/ -f json -o bandit-report.json || true
- name: Run Semgrep security scan code:
run: | name: Static analysis
semgrep --config=auto src/ --json --output=semgrep-report.json || true
- name: Upload security reports
uses: actions/upload-artifact@v3
with:
name: security-reports-${{ github.run_number }}
path: |
safety-report.json
safety-dev-report.json
bandit-report.json
semgrep-report.json
- name: Check for critical vulnerabilities
run: |
echo "Checking for critical vulnerabilities..."
# Check Safety results
if [ -f safety-report.json ]; then
critical_count=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null || echo "0")
if [ "$critical_count" -gt 0 ]; then
echo "Found $critical_count dependency vulnerabilities"
jq '.vulnerabilities[] | "- \(.package_name) \(.installed_version): \(.vulnerability_id)"' safety-report.json
else
echo "No dependency vulnerabilities found"
fi
fi
# Check Bandit results
if [ -f bandit-report.json ]; then
high_severity=$(jq '.results[] | select(.issue_severity == "HIGH") | length' bandit-report.json 2>/dev/null | wc -l)
if [ "$high_severity" -gt 0 ]; then
echo "Found $high_severity high-severity security issues"
else
echo "No high-severity security issues found"
fi
fi
# License compliance check
license-check:
name: License Compliance
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout code - uses: actions/checkout@v4
uses: actions/checkout@v4
with:
token: ${{ secrets.GITEA_TOKEN }}
- name: Set up Python - uses: actions/setup-python@v5
uses: actions/setup-python@v4
with: with:
python-version: ${{ env.PYTHON_VERSION }} python-version: ${{ env.PYTHON_VERSION }}
- name: Install pip-licenses - name: Install bandit
run: | run: |
python -m pip install --upgrade pip --root-user-action=ignore python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore pip-licenses pip install --root-user-action=ignore bandit
pip install --root-user-action=ignore -r requirements.txt
- name: Check licenses - name: bandit (HIGH fails; medium/low listed)
run: | run: |
echo "Checking dependency licenses..." bandit -r src/ -q --skip B104 -ll -ii || true
bandit -r src/ -q --skip B104 --severity-level high --confidence-level medium
licenses:
name: License report
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: pip
cache-dependency-path: requirements.txt
- name: Install
run: |
python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore -r requirements.txt pip-licenses
- name: Report
run: |
pip-licenses --format=markdown --with-urls --output-file=licenses.md
pip-licenses --format=json --output-file=licenses.json pip-licenses --format=json --output-file=licenses.json
pip-licenses --format=markdown --output-file=licenses.md echo "Copyleft licenses among runtime deps (informational):"
pip-licenses --format=plain | grep -iE 'GPL|AGPL|LGPL' || echo " none"
# Check for problematic licenses - uses: actions/upload-artifact@v3
problematic_licenses=("GPL" "AGPL" "LGPL")
for license in "${problematic_licenses[@]}"; do
if grep -i "$license" licenses.json; then
echo "Found potentially problematic license: $license"
fi
done
echo "License check completed"
- name: Upload license report
uses: actions/upload-artifact@v3
with: with:
name: license-report-${{ github.run_number }} name: licenses-${{ github.run_number }}
path: | path: |
licenses.json
licenses.md licenses.md
licenses.json
# Dependency update check
dependency-update:
name: Check for Dependency Updates
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
token: ${{ secrets.GITEA_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install pip-check-updates equivalent
run: |
python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore pip-review
- name: Check for outdated packages
run: |
echo "Checking for outdated packages..."
pip install --root-user-action=ignore -r requirements.txt
pip list --outdated --format=json > outdated-packages.json || true
if [ -s outdated-packages.json ]; then
echo "Outdated packages found:"
cat outdated-packages.json | jq -r '.[] | "- \(.name): \(.version) -> \(.latest_version)"'
else
echo "All packages are up to date"
fi
- name: Upload dependency reports
uses: actions/upload-artifact@v3
with:
name: dependency-reports-${{ github.run_number }}
path: |
outdated-packages.json
# Code quality metrics
code-quality:
name: Code Quality Metrics
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
token: ${{ secrets.GITEA_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Install quality tools
run: |
python -m pip install --upgrade pip --root-user-action=ignore
pip install --root-user-action=ignore radon xenon vulture
pip install --root-user-action=ignore -r requirements.txt
- name: Calculate code complexity
run: |
echo "Calculating code complexity..."
radon cc src/ --json > complexity-report.json
radon mi src/ --json > maintainability-report.json
echo "Complexity Summary:"
radon cc src/ --average
echo "Maintainability Summary:"
radon mi src/
- name: Find dead code
run: |
echo "Checking for dead code..."
vulture src/ --json > dead-code-report.json || true
- name: Check for code smells
run: |
echo "Checking for code smells..."
xenon --max-absolute B --max-modules A --max-average A src/ || true
- name: Upload quality reports
uses: actions/upload-artifact@v3
with:
name: code-quality-reports-${{ github.run_number }}
path: |
complexity-report.json
maintainability-report.json
dead-code-report.json
# Security summary
security-summary:
name: Security Summary
runs-on: ubuntu-latest
needs: [dependency-scan, license-check, code-quality]
if: always()
steps:
- name: Download all artifacts
uses: actions/download-artifact@v3
- name: Generate security summary
run: |
echo "# Security Scan Summary" > security-summary.md
echo "" >> security-summary.md
echo "**Scan Date:** $(date -u)" >> security-summary.md
echo "**Repository:** ${{ github.repository }}" >> security-summary.md
echo "**Commit:** ${{ github.sha }}" >> security-summary.md
echo "" >> security-summary.md
echo "## Results" >> security-summary.md
echo "" >> security-summary.md
# Dependency scan results
if [ -f security-reports-*/safety-report.json ]; then
vuln_count=$(jq '.vulnerabilities | length' security-reports-*/safety-report.json 2>/dev/null || echo "0")
if [ "$vuln_count" -eq 0 ]; then
echo "- Dependency Scan: No vulnerabilities found" >> security-summary.md
else
echo "- Dependency Scan: $vuln_count vulnerabilities found" >> security-summary.md
fi
else
echo "- Dependency Scan: Results not available" >> security-summary.md
fi
# Docker scan results (removed Trivy)
echo "- Docker Scan: Skipped (Trivy removed)" >> security-summary.md
# License check results
if [ -f license-report-*/licenses.json ]; then
echo "- License Check: Completed" >> security-summary.md
else
echo "- License Check: Results not available" >> security-summary.md
fi
# Code quality results
if [ -f code-quality-reports-*/complexity-report.json ]; then
echo "- Code Quality: Analyzed" >> security-summary.md
else
echo "- Code Quality: Results not available" >> security-summary.md
fi
echo "" >> security-summary.md
echo "## Detailed Reports" >> security-summary.md
echo "" >> security-summary.md
echo "Detailed reports are available in the workflow artifacts." >> security-summary.md
cat security-summary.md
- name: Upload security summary
uses: actions/upload-artifact@v3
with:
name: security-summary-${{ github.run_number }}
path: security-summary.md
+1 -1
View File
@@ -19,7 +19,7 @@ repos:
# Python code formatting with Black # Python code formatting with Black
- repo: https://github.com/psf/black - repo: https://github.com/psf/black
rev: 23.11.0 rev: 26.5.1
hooks: hooks:
- id: black - id: black
language_version: python3 language_version: python3
+13 -13
View File
@@ -34,23 +34,23 @@ classifiers = [
"Environment :: Web Environment", "Environment :: Web Environment",
"Framework :: FastAPI" "Framework :: FastAPI"
] ]
requires-python = ">=3.11" requires-python = ">=3.11,<3.12"
dependencies = [ dependencies = [
# Core dependencies # Core dependencies
"requests==2.31.0", "requests==2.34.2",
"schedule==1.2.0", "schedule==1.2.0",
"pandas==2.0.3", "pandas==2.0.3",
"numpy>=1.24,<2", "numpy>=1.24,<2",
# Flood forecasting (ML) # Flood forecasting (ML)
"scikit-learn==1.9.0", "scikit-learn==1.9.0",
# Web API framework # Web API framework
"fastapi==0.104.1", "fastapi==0.141.1",
"uvicorn[standard]==0.24.0", "uvicorn[standard]==0.52.4",
"pydantic==2.5.0", "pydantic==2.13.5",
# Database adapters # Database adapters
"sqlalchemy==2.0.23", "sqlalchemy==2.0.23",
"influxdb==5.3.1", "influxdb==5.3.1",
"pymysql==1.1.0", "pymysql==1.2.0",
"psycopg2-binary==2.9.9", "psycopg2-binary==2.9.9",
# Monitoring and metrics # Monitoring and metrics
"psutil==5.9.6" "psutil==5.9.6"
@@ -59,11 +59,11 @@ dependencies = [
[project.optional-dependencies] [project.optional-dependencies]
dev = [ dev = [
# Testing # Testing
"pytest==7.4.3", "pytest==9.1.1",
"pytest-cov==4.1.0", "pytest-cov==4.1.0",
"pytest-asyncio==0.21.1", "pytest-asyncio==0.21.1",
# Code formatting and linting # Code formatting and linting
"black==23.11.0", "black==26.5.1",
"flake8==6.1.0", "flake8==6.1.0",
"isort==5.12.0", "isort==5.12.0",
"mypy==1.7.1", "mypy==1.7.1",
@@ -73,7 +73,7 @@ dev = [
"ipython==8.17.2", "ipython==8.17.2",
"jupyter==1.0.0", "jupyter==1.0.0",
# Type stubs # Type stubs
"types-requests==2.31.0.10", "types-requests==2.33.0.20260906",
"types-python-dateutil==2.8.19.14" "types-python-dateutil==2.8.19.14"
] ]
docs = [ docs = [
@@ -83,7 +83,7 @@ docs = [
] ]
all = [ all = [
"influxdb==5.3.1", "influxdb==5.3.1",
"pymysql==1.1.0", "pymysql==1.2.0",
"psycopg2-binary==2.9.9" "psycopg2-binary==2.9.9"
] ]
@@ -100,11 +100,11 @@ Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/
[dependency-groups] [dependency-groups]
dev = [ dev = [
# Testing # Testing
"pytest==7.4.3", "pytest==9.1.1",
"pytest-cov==4.1.0", "pytest-cov==4.1.0",
"pytest-asyncio==0.21.1", "pytest-asyncio==0.21.1",
# Code formatting and linting # Code formatting and linting
"black==23.11.0", "black==26.5.1",
"flake8==6.1.0", "flake8==6.1.0",
"isort==5.12.0", "isort==5.12.0",
"mypy==1.7.1", "mypy==1.7.1",
@@ -114,7 +114,7 @@ dev = [
"ipython==8.17.2", "ipython==8.17.2",
"jupyter==1.0.0", "jupyter==1.0.0",
# Type stubs # Type stubs
"types-requests==2.31.0.10", "types-requests==2.33.0.20260906",
"types-python-dateutil==2.8.19.14", "types-python-dateutil==2.8.19.14",
# Documentation # Documentation
"sphinx==7.2.6", "sphinx==7.2.6",
+3 -3
View File
@@ -2,12 +2,12 @@
-r requirements.txt -r requirements.txt
# Testing # Testing
pytest==7.4.3 pytest==9.1.1
pytest-cov==4.1.0 pytest-cov==4.1.0
pytest-asyncio==0.21.1 pytest-asyncio==0.21.1
# Code formatting and linting # Code formatting and linting
black==23.11.0 black==26.5.1
flake8==6.1.0 flake8==6.1.0
isort==5.12.0 isort==5.12.0
mypy==1.7.1 mypy==1.7.1
@@ -25,5 +25,5 @@ ipython==8.17.2
jupyter==1.0.0 jupyter==1.0.0
# Type stubs # Type stubs
types-requests==2.31.0.10 types-requests==2.33.0.20260906
types-python-dateutil==2.8.19.14 types-python-dateutil==2.8.19.14
+7 -7
View File
@@ -1,5 +1,5 @@
# Core dependencies # Core dependencies
requests==2.31.0 requests==2.34.2
schedule==1.2.0 schedule==1.2.0
pandas==2.0.3 pandas==2.0.3
numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
@@ -8,23 +8,23 @@ numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
scikit-learn==1.9.0 scikit-learn==1.9.0
# Web API framework # Web API framework
fastapi==0.104.1 fastapi==0.141.1
uvicorn[standard]==0.24.0 uvicorn[standard]==0.52.4
pydantic==2.5.0 pydantic==2.13.5
# Database adapters # Database adapters
sqlalchemy==2.0.23 sqlalchemy==2.0.23
influxdb==5.3.1 influxdb==5.3.1
pymysql==1.1.0 pymysql==1.2.0
psycopg2-binary==2.9.9 psycopg2-binary==2.9.9
# Monitoring and metrics # Monitoring and metrics
psutil==5.9.6 psutil==5.9.6
# Development dependencies (optional) # Development dependencies (optional)
pytest==7.4.3 pytest==9.1.1
pytest-cov==4.1.0 pytest-cov==4.1.0
black==23.11.0 black==26.5.1
flake8==6.1.0 flake8==6.1.0
mypy==1.7.1 mypy==1.7.1
pre-commit==3.5.0 pre-commit==3.5.0
+26 -22
View File
@@ -139,12 +139,16 @@ class InfluxDBAdapter(DatabaseAdapter):
"time": measurement["timestamp"].isoformat(), "time": measurement["timestamp"].isoformat(),
"fields": { "fields": {
"water_level": float(measurement["water_level"]), "water_level": float(measurement["water_level"]),
"discharge": float(measurement["discharge"]) "discharge": (
float(measurement["discharge"])
if measurement.get("discharge") is not None if measurement.get("discharge") is not None
else None, else None
"discharge_percent": float(measurement["discharge_percent"]) ),
"discharge_percent": (
float(measurement["discharge_percent"])
if measurement.get("discharge_percent") if measurement.get("discharge_percent")
else None, else None
),
}, },
} }
points.append(point) points.append(point)
@@ -551,13 +555,13 @@ class SQLAdapter(DatabaseAdapter):
"station_code": row[1], "station_code": row[1],
"station_name_en": row[2], "station_name_en": row[2],
"station_name_th": row[3], "station_name_th": row[3],
"water_level": float(row[4]) "water_level": (
if row[4] is not None float(row[4]) if row[4] is not None else None
else None, ),
"discharge": float(row[5]) if row[5] is not None else None, "discharge": float(row[5]) if row[5] is not None else None,
"discharge_percent": float(row[6]) "discharge_percent": (
if row[6] is not None float(row[6]) if row[6] is not None else None
else None, ),
"status": row[7], "status": row[7],
} }
) )
@@ -611,13 +615,13 @@ class SQLAdapter(DatabaseAdapter):
"station_code": row[1], "station_code": row[1],
"station_name_en": row[2], "station_name_en": row[2],
"station_name_th": row[3], "station_name_th": row[3],
"water_level": float(row[4]) "water_level": (
if row[4] is not None float(row[4]) if row[4] is not None else None
else None, ),
"discharge": float(row[5]) if row[5] is not None else None, "discharge": float(row[5]) if row[5] is not None else None,
"discharge_percent": float(row[6]) "discharge_percent": (
if row[6] is not None float(row[6]) if row[6] is not None else None
else None, ),
"status": row[7], "status": row[7],
} }
) )
@@ -666,13 +670,13 @@ class SQLAdapter(DatabaseAdapter):
"station_id": row[1], "station_id": row[1],
"station_code": row[2] or f"Station_{row[1]}", "station_code": row[2] or f"Station_{row[1]}",
"station_name_th": row[3] or f"Station {row[1]}", "station_name_th": row[3] or f"Station {row[1]}",
"water_level": float(row[4]) "water_level": (
if row[4] is not None float(row[4]) if row[4] is not None else None
else None, ),
"discharge": float(row[5]) if row[5] is not None else None, "discharge": float(row[5]) if row[5] is not None else None,
"discharge_percent": float(row[6]) "discharge_percent": (
if row[6] is not None float(row[6]) if row[6] is not None else None
else None, ),
"status": row[7], "status": row[7],
} }
) )
+3 -3
View File
@@ -125,9 +125,9 @@ class DatabaseHealthCheck(HealthCheck):
"message": "Database connection OK", "message": "Database connection OK",
"details": { "details": {
"latest_data_count": len(latest_data), "latest_data_count": len(latest_data),
"latest_timestamp": str(latest_data[0].get("timestamp")) "latest_timestamp": (
if latest_data str(latest_data[0].get("timestamp")) if latest_data else None
else None, ),
}, },
} }
+3 -3
View File
@@ -207,9 +207,9 @@ def fill_from_hii(
"timestamp": missing["timestamp"], "timestamp": missing["timestamp"],
"station_code": code, "station_code": code,
"water_level": missing["wl_msl"] - offset, "water_level": missing["wl_msl"] - offset,
"discharge": missing["discharge"] "discharge": (
if code in _HII_EXACT_MIRRORS missing["discharge"] if code in _HII_EXACT_MIRRORS else float("nan")
else float("nan"), ),
} }
) )
fills.append(fill) fills.append(fill)
+6 -6
View File
@@ -320,9 +320,9 @@ def train_station(
skipped_heads, skipped_heads,
) )
else: else:
skipped_heads[ skipped_heads[head_key] = (
head_key f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})" )
heads[head_key] = clf heads[head_key] = clf
if not skip_eval: if not skip_eval:
@@ -417,9 +417,9 @@ def train_station(
if clf is not None: if clf is not None:
skipped_heads.pop(head_key, None) skipped_heads.pop(head_key, None)
else: else:
skipped_heads[ skipped_heads[head_key] = (
head_key f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})" )
final_heads[head_key] = None final_heads[head_key] = None
# v4 = + Mae Ngat dam features; v3 = rise + rain; v2 = rise target only # v4 = + Mae Ngat dam features; v3 = rise + rain; v2 = rise target only
+5 -7
View File
@@ -51,8 +51,7 @@ class PostgresHistory:
if start >= end: if start >= end:
raise ValueError("start must be before end") raise ValueError("start must be before end")
query = text( query = text("""
"""
SELECT m.timestamp, s.station_code, m.water_level, SELECT m.timestamp, s.station_code, m.water_level,
m.discharge, m.discharge_percent m.discharge, m.discharge_percent
FROM water_measurements m FROM water_measurements m
@@ -62,8 +61,7 @@ class PostgresHistory:
AND m.timestamp <= :end_time AND m.timestamp <= :end_time
ORDER BY m.timestamp ASC ORDER BY m.timestamp ASC
LIMIT :limit LIMIT :limit
""" """)
)
with self.engine.connect() as connection: with self.engine.connect() as connection:
rows = connection.execute( rows = connection.execute(
query, query,
@@ -91,9 +89,9 @@ class PostgresHistory:
"station_code": station_code, "station_code": station_code,
"water_level": water_level, "water_level": water_level,
"discharge": discharge, "discharge": discharge,
"discharge_percent": float(row[4]) "discharge_percent": (
if row[4] is not None float(row[4]) if row[4] is not None else None
else None, ),
} }
) )
return result return result
+4 -2
View File
@@ -173,8 +173,10 @@ class RequestTracker:
"failed_requests": self.failed_requests, "failed_requests": self.failed_requests,
"success_rate": self.successful_requests / self.total_requests, "success_rate": self.successful_requests / self.total_requests,
"average_response_time": self.total_response_time / self.total_requests, "average_response_time": self.total_response_time / self.total_requests,
"last_request_time": self.last_request_time.isoformat() "last_request_time": (
self.last_request_time.isoformat()
if self.last_request_time if self.last_request_time
else None, else None
),
"error_breakdown": dict(self.error_count_by_type), "error_breakdown": dict(self.error_count_by_type),
} }
+6
View File
@@ -323,9 +323,11 @@ class RidReservoirStore:
if not preserve_cols: if not preserve_cols:
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})" return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
updates = ", ".join( updates = ", ".join(
(
f"{c} = COALESCE(excluded.{c}, {table}.{c})" f"{c} = COALESCE(excluded.{c}, {table}.{c})"
if c in preserve_cols if c in preserve_cols
else f"{c} = excluded.{c}" else f"{c} = excluded.{c}"
)
for c in value_cols for c in value_cols
) )
return ( return (
@@ -334,9 +336,11 @@ class RidReservoirStore:
) )
if self.db_type == "postgresql": if self.db_type == "postgresql":
updates = ", ".join( updates = ", ".join(
(
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})" f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
if c in preserve_cols if c in preserve_cols
else f"{c} = EXCLUDED.{c}" else f"{c} = EXCLUDED.{c}"
)
for c in value_cols for c in value_cols
) )
return ( return (
@@ -344,9 +348,11 @@ class RidReservoirStore:
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}" f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
) )
updates = ", ".join( updates = ", ".join(
(
f"{c} = COALESCE(VALUES({c}), {c})" f"{c} = COALESCE(VALUES({c}), {c})"
if c in preserve_cols if c in preserve_cols
else f"{c} = VALUES({c})" else f"{c} = VALUES({c})"
)
for c in value_cols for c in value_cols
) )
return ( return (
Generated
+129 -713
View File
File diff suppressed because it is too large Load Diff