security: pip-audit + bandit gates that can fail; patch 29 known CVEs
security.yml previously ran safety/bandit/semgrep with `|| true` and could not go red. Now: pip-audit on requirements.txt is a hard gate (dev deps reported only), bandit HIGH fails (B104 bind-all skipped: intended behind Cloudflare/Caddy), pip-licenses uploaded as a report. Weekly + on dependency/source changes. Running it locally found 29 advisories, all in pinned-and-forgotten runtime deps: starlette 0.27 (7, incl. Host-header path confusion and form DoS), fastapi 0.104, requests 2.31 (3), pymysql 1.1. Bumped to current: fastapi 0.141.1 / starlette 1.6.0, pydantic 2.13.5, uvicorn 0.52.4, requests 2.34.2, pymysql 1.2.0; dev: pytest 9.1.1, black 26.5.1. pip-audit is now clean. requires-python narrowed to 3.11 (the truth: psycopg2-binary 2.9.9 fails on 3.13; pandas 2.0.3 has no 3.12 wheels). Full suite passes; API smoke-tested (health, stations, forecast, history, stats, docs, openapi) on the new stack. black 26 reformatted 8 files.
This commit is contained in:
@@ -38,7 +38,7 @@ jobs:
|
|||||||
- name: Install tools
|
- name: Install tools
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
python -m pip install --upgrade pip --root-user-action=ignore
|
||||||
pip install --root-user-action=ignore black==23.11.0 isort==5.12.0 flake8==6.1.0
|
pip install --root-user-action=ignore black==26.5.1 isort==5.12.0 flake8==6.1.0
|
||||||
|
|
||||||
- name: black
|
- name: black
|
||||||
run: black --check --diff src/ *.py
|
run: black --check --diff src/ *.py
|
||||||
@@ -67,7 +67,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
python -m pip install --upgrade pip --root-user-action=ignore
|
||||||
pip install --root-user-action=ignore -r requirements.txt
|
pip install --root-user-action=ignore -r requirements.txt
|
||||||
pip install --root-user-action=ignore pytest==7.4.3 pytest-asyncio==0.21.1
|
pip install --root-user-action=ignore pytest==9.1.1 pytest-asyncio==0.21.1
|
||||||
|
|
||||||
- name: pytest
|
- name: pytest
|
||||||
env:
|
env:
|
||||||
|
|||||||
+65
-254
@@ -1,293 +1,104 @@
|
|||||||
name: Security & Dependency Updates
|
name: Security
|
||||||
|
|
||||||
|
# Two gates that can actually fail, plus one report:
|
||||||
|
# - pip-audit against requirements.txt: any known vulnerability in a runtime
|
||||||
|
# dependency fails the job (dev-only tools are reported, not gated)
|
||||||
|
# - bandit on src/: HIGH severity findings fail; medium/low are listed.
|
||||||
|
# B104 (bind 0.0.0.0) is skipped: the service is meant to listen on all
|
||||||
|
# interfaces behind Cloudflare/Caddy.
|
||||||
|
# - pip-licenses report as an artifact (informational; the project is MIT
|
||||||
|
# and its runtime deps are MIT/BSD/Apache/PSF)
|
||||||
|
# The old file ran safety/bandit/semgrep with `|| true` and could not go red.
|
||||||
|
|
||||||
on:
|
on:
|
||||||
schedule:
|
schedule:
|
||||||
# Run security scans daily at 3 AM UTC
|
- cron: "0 3 * * 1" # weekly, Monday 03:00 UTC
|
||||||
- cron: "0 3 * * *"
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
push:
|
push:
|
||||||
paths:
|
paths:
|
||||||
- "requirements*.txt"
|
- "requirements*.txt"
|
||||||
- "Dockerfile"
|
- "pyproject.toml"
|
||||||
|
- "uv.lock"
|
||||||
|
- "src/**/*.py"
|
||||||
- ".gitea/workflows/security.yml"
|
- ".gitea/workflows/security.yml"
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- "requirements*.txt"
|
||||||
|
- "pyproject.toml"
|
||||||
|
- "src/**/*.py"
|
||||||
|
|
||||||
env:
|
env:
|
||||||
PYTHON_VERSION: "3.11"
|
PYTHON_VERSION: "3.11"
|
||||||
# GitHub token for better rate limits and authentication
|
|
||||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Dependency vulnerability scan
|
dependencies:
|
||||||
dependency-scan:
|
name: Dependency vulnerabilities
|
||||||
name: Dependency Security Scan
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- uses: actions/checkout@v4
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
|
|
||||||
- name: Set up Python
|
- uses: actions/setup-python@v5
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
with:
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install pip-audit
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
python -m pip install --upgrade pip --root-user-action=ignore
|
||||||
pip install --root-user-action=ignore safety bandit semgrep
|
pip install --root-user-action=ignore pip-audit
|
||||||
|
|
||||||
- name: Run Safety check
|
- name: Runtime dependencies (gate)
|
||||||
run: |
|
run: pip-audit -r requirements.txt --strict --desc on
|
||||||
safety check -r requirements.txt --json --output safety-report.json || true
|
|
||||||
safety check -r requirements-dev.txt --json --output safety-dev-report.json || true
|
|
||||||
|
|
||||||
- name: Run Bandit security scan
|
- name: Dev dependencies (report only)
|
||||||
run: |
|
run: pip-audit -r requirements-dev.txt --desc on || echo "::warning::dev-only dependency advisories above"
|
||||||
bandit -r src/ -f json -o bandit-report.json || true
|
|
||||||
|
|
||||||
- name: Run Semgrep security scan
|
code:
|
||||||
run: |
|
name: Static analysis
|
||||||
semgrep --config=auto src/ --json --output=semgrep-report.json || true
|
|
||||||
|
|
||||||
- name: Upload security reports
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: security-reports-${{ github.run_number }}
|
|
||||||
path: |
|
|
||||||
safety-report.json
|
|
||||||
safety-dev-report.json
|
|
||||||
bandit-report.json
|
|
||||||
semgrep-report.json
|
|
||||||
|
|
||||||
- name: Check for critical vulnerabilities
|
|
||||||
run: |
|
|
||||||
echo "Checking for critical vulnerabilities..."
|
|
||||||
|
|
||||||
# Check Safety results
|
|
||||||
if [ -f safety-report.json ]; then
|
|
||||||
critical_count=$(jq '.vulnerabilities | length' safety-report.json 2>/dev/null || echo "0")
|
|
||||||
if [ "$critical_count" -gt 0 ]; then
|
|
||||||
echo "Found $critical_count dependency vulnerabilities"
|
|
||||||
jq '.vulnerabilities[] | "- \(.package_name) \(.installed_version): \(.vulnerability_id)"' safety-report.json
|
|
||||||
else
|
|
||||||
echo "No dependency vulnerabilities found"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check Bandit results
|
|
||||||
if [ -f bandit-report.json ]; then
|
|
||||||
high_severity=$(jq '.results[] | select(.issue_severity == "HIGH") | length' bandit-report.json 2>/dev/null | wc -l)
|
|
||||||
if [ "$high_severity" -gt 0 ]; then
|
|
||||||
echo "Found $high_severity high-severity security issues"
|
|
||||||
else
|
|
||||||
echo "No high-severity security issues found"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# License compliance check
|
|
||||||
license-check:
|
|
||||||
name: License Compliance
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- uses: actions/checkout@v4
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
|
|
||||||
- name: Set up Python
|
- uses: actions/setup-python@v5
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
with:
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
|
||||||
- name: Install pip-licenses
|
- name: Install bandit
|
||||||
run: |
|
run: |
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
python -m pip install --upgrade pip --root-user-action=ignore
|
||||||
pip install --root-user-action=ignore pip-licenses
|
pip install --root-user-action=ignore bandit
|
||||||
pip install --root-user-action=ignore -r requirements.txt
|
|
||||||
|
|
||||||
- name: Check licenses
|
- name: bandit (HIGH fails; medium/low listed)
|
||||||
run: |
|
run: |
|
||||||
echo "Checking dependency licenses..."
|
bandit -r src/ -q --skip B104 -ll -ii || true
|
||||||
|
bandit -r src/ -q --skip B104 --severity-level high --confidence-level medium
|
||||||
|
|
||||||
|
licenses:
|
||||||
|
name: License report
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ env.PYTHON_VERSION }}
|
||||||
|
cache: pip
|
||||||
|
cache-dependency-path: requirements.txt
|
||||||
|
|
||||||
|
- name: Install
|
||||||
|
run: |
|
||||||
|
python -m pip install --upgrade pip --root-user-action=ignore
|
||||||
|
pip install --root-user-action=ignore -r requirements.txt pip-licenses
|
||||||
|
|
||||||
|
- name: Report
|
||||||
|
run: |
|
||||||
|
pip-licenses --format=markdown --with-urls --output-file=licenses.md
|
||||||
pip-licenses --format=json --output-file=licenses.json
|
pip-licenses --format=json --output-file=licenses.json
|
||||||
pip-licenses --format=markdown --output-file=licenses.md
|
echo "Copyleft licenses among runtime deps (informational):"
|
||||||
|
pip-licenses --format=plain | grep -iE 'GPL|AGPL|LGPL' || echo " none"
|
||||||
|
|
||||||
# Check for problematic licenses
|
- uses: actions/upload-artifact@v3
|
||||||
problematic_licenses=("GPL" "AGPL" "LGPL")
|
|
||||||
|
|
||||||
for license in "${problematic_licenses[@]}"; do
|
|
||||||
if grep -i "$license" licenses.json; then
|
|
||||||
echo "Found potentially problematic license: $license"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "License check completed"
|
|
||||||
|
|
||||||
- name: Upload license report
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
with:
|
||||||
name: license-report-${{ github.run_number }}
|
name: licenses-${{ github.run_number }}
|
||||||
path: |
|
path: |
|
||||||
licenses.json
|
|
||||||
licenses.md
|
licenses.md
|
||||||
|
licenses.json
|
||||||
# Dependency update check
|
|
||||||
dependency-update:
|
|
||||||
name: Check for Dependency Updates
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
|
|
||||||
- name: Install pip-check-updates equivalent
|
|
||||||
run: |
|
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
|
||||||
pip install --root-user-action=ignore pip-review
|
|
||||||
|
|
||||||
- name: Check for outdated packages
|
|
||||||
run: |
|
|
||||||
echo "Checking for outdated packages..."
|
|
||||||
pip install --root-user-action=ignore -r requirements.txt
|
|
||||||
pip list --outdated --format=json > outdated-packages.json || true
|
|
||||||
|
|
||||||
if [ -s outdated-packages.json ]; then
|
|
||||||
echo "Outdated packages found:"
|
|
||||||
cat outdated-packages.json | jq -r '.[] | "- \(.name): \(.version) -> \(.latest_version)"'
|
|
||||||
else
|
|
||||||
echo "All packages are up to date"
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Upload dependency reports
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: dependency-reports-${{ github.run_number }}
|
|
||||||
path: |
|
|
||||||
outdated-packages.json
|
|
||||||
|
|
||||||
# Code quality metrics
|
|
||||||
code-quality:
|
|
||||||
name: Code Quality Metrics
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
token: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: ${{ env.PYTHON_VERSION }}
|
|
||||||
|
|
||||||
- name: Install quality tools
|
|
||||||
run: |
|
|
||||||
python -m pip install --upgrade pip --root-user-action=ignore
|
|
||||||
pip install --root-user-action=ignore radon xenon vulture
|
|
||||||
pip install --root-user-action=ignore -r requirements.txt
|
|
||||||
|
|
||||||
- name: Calculate code complexity
|
|
||||||
run: |
|
|
||||||
echo "Calculating code complexity..."
|
|
||||||
radon cc src/ --json > complexity-report.json
|
|
||||||
radon mi src/ --json > maintainability-report.json
|
|
||||||
|
|
||||||
echo "Complexity Summary:"
|
|
||||||
radon cc src/ --average
|
|
||||||
|
|
||||||
echo "Maintainability Summary:"
|
|
||||||
radon mi src/
|
|
||||||
|
|
||||||
- name: Find dead code
|
|
||||||
run: |
|
|
||||||
echo "Checking for dead code..."
|
|
||||||
vulture src/ --json > dead-code-report.json || true
|
|
||||||
|
|
||||||
- name: Check for code smells
|
|
||||||
run: |
|
|
||||||
echo "Checking for code smells..."
|
|
||||||
xenon --max-absolute B --max-modules A --max-average A src/ || true
|
|
||||||
|
|
||||||
- name: Upload quality reports
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: code-quality-reports-${{ github.run_number }}
|
|
||||||
path: |
|
|
||||||
complexity-report.json
|
|
||||||
maintainability-report.json
|
|
||||||
dead-code-report.json
|
|
||||||
|
|
||||||
# Security summary
|
|
||||||
security-summary:
|
|
||||||
name: Security Summary
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: [dependency-scan, license-check, code-quality]
|
|
||||||
if: always()
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Download all artifacts
|
|
||||||
uses: actions/download-artifact@v3
|
|
||||||
|
|
||||||
- name: Generate security summary
|
|
||||||
run: |
|
|
||||||
echo "# Security Scan Summary" > security-summary.md
|
|
||||||
echo "" >> security-summary.md
|
|
||||||
echo "**Scan Date:** $(date -u)" >> security-summary.md
|
|
||||||
echo "**Repository:** ${{ github.repository }}" >> security-summary.md
|
|
||||||
echo "**Commit:** ${{ github.sha }}" >> security-summary.md
|
|
||||||
echo "" >> security-summary.md
|
|
||||||
|
|
||||||
echo "## Results" >> security-summary.md
|
|
||||||
echo "" >> security-summary.md
|
|
||||||
|
|
||||||
# Dependency scan results
|
|
||||||
if [ -f security-reports-*/safety-report.json ]; then
|
|
||||||
vuln_count=$(jq '.vulnerabilities | length' security-reports-*/safety-report.json 2>/dev/null || echo "0")
|
|
||||||
if [ "$vuln_count" -eq 0 ]; then
|
|
||||||
echo "- Dependency Scan: No vulnerabilities found" >> security-summary.md
|
|
||||||
else
|
|
||||||
echo "- Dependency Scan: $vuln_count vulnerabilities found" >> security-summary.md
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "- Dependency Scan: Results not available" >> security-summary.md
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Docker scan results (removed Trivy)
|
|
||||||
echo "- Docker Scan: Skipped (Trivy removed)" >> security-summary.md
|
|
||||||
|
|
||||||
# License check results
|
|
||||||
if [ -f license-report-*/licenses.json ]; then
|
|
||||||
echo "- License Check: Completed" >> security-summary.md
|
|
||||||
else
|
|
||||||
echo "- License Check: Results not available" >> security-summary.md
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Code quality results
|
|
||||||
if [ -f code-quality-reports-*/complexity-report.json ]; then
|
|
||||||
echo "- Code Quality: Analyzed" >> security-summary.md
|
|
||||||
else
|
|
||||||
echo "- Code Quality: Results not available" >> security-summary.md
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "" >> security-summary.md
|
|
||||||
echo "## Detailed Reports" >> security-summary.md
|
|
||||||
echo "" >> security-summary.md
|
|
||||||
echo "Detailed reports are available in the workflow artifacts." >> security-summary.md
|
|
||||||
|
|
||||||
cat security-summary.md
|
|
||||||
|
|
||||||
- name: Upload security summary
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: security-summary-${{ github.run_number }}
|
|
||||||
path: security-summary.md
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ repos:
|
|||||||
|
|
||||||
# Python code formatting with Black
|
# Python code formatting with Black
|
||||||
- repo: https://github.com/psf/black
|
- repo: https://github.com/psf/black
|
||||||
rev: 23.11.0
|
rev: 26.5.1
|
||||||
hooks:
|
hooks:
|
||||||
- id: black
|
- id: black
|
||||||
language_version: python3
|
language_version: python3
|
||||||
|
|||||||
+13
-13
@@ -34,23 +34,23 @@ classifiers = [
|
|||||||
"Environment :: Web Environment",
|
"Environment :: Web Environment",
|
||||||
"Framework :: FastAPI"
|
"Framework :: FastAPI"
|
||||||
]
|
]
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11,<3.12"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
# Core dependencies
|
# Core dependencies
|
||||||
"requests==2.31.0",
|
"requests==2.34.2",
|
||||||
"schedule==1.2.0",
|
"schedule==1.2.0",
|
||||||
"pandas==2.0.3",
|
"pandas==2.0.3",
|
||||||
"numpy>=1.24,<2",
|
"numpy>=1.24,<2",
|
||||||
# Flood forecasting (ML)
|
# Flood forecasting (ML)
|
||||||
"scikit-learn==1.9.0",
|
"scikit-learn==1.9.0",
|
||||||
# Web API framework
|
# Web API framework
|
||||||
"fastapi==0.104.1",
|
"fastapi==0.141.1",
|
||||||
"uvicorn[standard]==0.24.0",
|
"uvicorn[standard]==0.52.4",
|
||||||
"pydantic==2.5.0",
|
"pydantic==2.13.5",
|
||||||
# Database adapters
|
# Database adapters
|
||||||
"sqlalchemy==2.0.23",
|
"sqlalchemy==2.0.23",
|
||||||
"influxdb==5.3.1",
|
"influxdb==5.3.1",
|
||||||
"pymysql==1.1.0",
|
"pymysql==1.2.0",
|
||||||
"psycopg2-binary==2.9.9",
|
"psycopg2-binary==2.9.9",
|
||||||
# Monitoring and metrics
|
# Monitoring and metrics
|
||||||
"psutil==5.9.6"
|
"psutil==5.9.6"
|
||||||
@@ -59,11 +59,11 @@ dependencies = [
|
|||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
dev = [
|
dev = [
|
||||||
# Testing
|
# Testing
|
||||||
"pytest==7.4.3",
|
"pytest==9.1.1",
|
||||||
"pytest-cov==4.1.0",
|
"pytest-cov==4.1.0",
|
||||||
"pytest-asyncio==0.21.1",
|
"pytest-asyncio==0.21.1",
|
||||||
# Code formatting and linting
|
# Code formatting and linting
|
||||||
"black==23.11.0",
|
"black==26.5.1",
|
||||||
"flake8==6.1.0",
|
"flake8==6.1.0",
|
||||||
"isort==5.12.0",
|
"isort==5.12.0",
|
||||||
"mypy==1.7.1",
|
"mypy==1.7.1",
|
||||||
@@ -73,7 +73,7 @@ dev = [
|
|||||||
"ipython==8.17.2",
|
"ipython==8.17.2",
|
||||||
"jupyter==1.0.0",
|
"jupyter==1.0.0",
|
||||||
# Type stubs
|
# Type stubs
|
||||||
"types-requests==2.31.0.10",
|
"types-requests==2.33.0.20260906",
|
||||||
"types-python-dateutil==2.8.19.14"
|
"types-python-dateutil==2.8.19.14"
|
||||||
]
|
]
|
||||||
docs = [
|
docs = [
|
||||||
@@ -83,7 +83,7 @@ docs = [
|
|||||||
]
|
]
|
||||||
all = [
|
all = [
|
||||||
"influxdb==5.3.1",
|
"influxdb==5.3.1",
|
||||||
"pymysql==1.1.0",
|
"pymysql==1.2.0",
|
||||||
"psycopg2-binary==2.9.9"
|
"psycopg2-binary==2.9.9"
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -100,11 +100,11 @@ Documentation = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/
|
|||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
# Testing
|
# Testing
|
||||||
"pytest==7.4.3",
|
"pytest==9.1.1",
|
||||||
"pytest-cov==4.1.0",
|
"pytest-cov==4.1.0",
|
||||||
"pytest-asyncio==0.21.1",
|
"pytest-asyncio==0.21.1",
|
||||||
# Code formatting and linting
|
# Code formatting and linting
|
||||||
"black==23.11.0",
|
"black==26.5.1",
|
||||||
"flake8==6.1.0",
|
"flake8==6.1.0",
|
||||||
"isort==5.12.0",
|
"isort==5.12.0",
|
||||||
"mypy==1.7.1",
|
"mypy==1.7.1",
|
||||||
@@ -114,7 +114,7 @@ dev = [
|
|||||||
"ipython==8.17.2",
|
"ipython==8.17.2",
|
||||||
"jupyter==1.0.0",
|
"jupyter==1.0.0",
|
||||||
# Type stubs
|
# Type stubs
|
||||||
"types-requests==2.31.0.10",
|
"types-requests==2.33.0.20260906",
|
||||||
"types-python-dateutil==2.8.19.14",
|
"types-python-dateutil==2.8.19.14",
|
||||||
# Documentation
|
# Documentation
|
||||||
"sphinx==7.2.6",
|
"sphinx==7.2.6",
|
||||||
|
|||||||
@@ -2,12 +2,12 @@
|
|||||||
-r requirements.txt
|
-r requirements.txt
|
||||||
|
|
||||||
# Testing
|
# Testing
|
||||||
pytest==7.4.3
|
pytest==9.1.1
|
||||||
pytest-cov==4.1.0
|
pytest-cov==4.1.0
|
||||||
pytest-asyncio==0.21.1
|
pytest-asyncio==0.21.1
|
||||||
|
|
||||||
# Code formatting and linting
|
# Code formatting and linting
|
||||||
black==23.11.0
|
black==26.5.1
|
||||||
flake8==6.1.0
|
flake8==6.1.0
|
||||||
isort==5.12.0
|
isort==5.12.0
|
||||||
mypy==1.7.1
|
mypy==1.7.1
|
||||||
@@ -25,5 +25,5 @@ ipython==8.17.2
|
|||||||
jupyter==1.0.0
|
jupyter==1.0.0
|
||||||
|
|
||||||
# Type stubs
|
# Type stubs
|
||||||
types-requests==2.31.0.10
|
types-requests==2.33.0.20260906
|
||||||
types-python-dateutil==2.8.19.14
|
types-python-dateutil==2.8.19.14
|
||||||
+7
-7
@@ -1,5 +1,5 @@
|
|||||||
# Core dependencies
|
# Core dependencies
|
||||||
requests==2.31.0
|
requests==2.34.2
|
||||||
schedule==1.2.0
|
schedule==1.2.0
|
||||||
pandas==2.0.3
|
pandas==2.0.3
|
||||||
numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
||||||
@@ -8,23 +8,23 @@ numpy>=1.24,<2 # pandas 2.0.3 wheels are ABI-incompatible with numpy 2.x
|
|||||||
scikit-learn==1.9.0
|
scikit-learn==1.9.0
|
||||||
|
|
||||||
# Web API framework
|
# Web API framework
|
||||||
fastapi==0.104.1
|
fastapi==0.141.1
|
||||||
uvicorn[standard]==0.24.0
|
uvicorn[standard]==0.52.4
|
||||||
pydantic==2.5.0
|
pydantic==2.13.5
|
||||||
|
|
||||||
# Database adapters
|
# Database adapters
|
||||||
sqlalchemy==2.0.23
|
sqlalchemy==2.0.23
|
||||||
influxdb==5.3.1
|
influxdb==5.3.1
|
||||||
pymysql==1.1.0
|
pymysql==1.2.0
|
||||||
psycopg2-binary==2.9.9
|
psycopg2-binary==2.9.9
|
||||||
|
|
||||||
# Monitoring and metrics
|
# Monitoring and metrics
|
||||||
psutil==5.9.6
|
psutil==5.9.6
|
||||||
|
|
||||||
# Development dependencies (optional)
|
# Development dependencies (optional)
|
||||||
pytest==7.4.3
|
pytest==9.1.1
|
||||||
pytest-cov==4.1.0
|
pytest-cov==4.1.0
|
||||||
black==23.11.0
|
black==26.5.1
|
||||||
flake8==6.1.0
|
flake8==6.1.0
|
||||||
mypy==1.7.1
|
mypy==1.7.1
|
||||||
pre-commit==3.5.0
|
pre-commit==3.5.0
|
||||||
|
|||||||
+28
-24
@@ -139,12 +139,16 @@ class InfluxDBAdapter(DatabaseAdapter):
|
|||||||
"time": measurement["timestamp"].isoformat(),
|
"time": measurement["timestamp"].isoformat(),
|
||||||
"fields": {
|
"fields": {
|
||||||
"water_level": float(measurement["water_level"]),
|
"water_level": float(measurement["water_level"]),
|
||||||
"discharge": float(measurement["discharge"])
|
"discharge": (
|
||||||
if measurement.get("discharge") is not None
|
float(measurement["discharge"])
|
||||||
else None,
|
if measurement.get("discharge") is not None
|
||||||
"discharge_percent": float(measurement["discharge_percent"])
|
else None
|
||||||
if measurement.get("discharge_percent")
|
),
|
||||||
else None,
|
"discharge_percent": (
|
||||||
|
float(measurement["discharge_percent"])
|
||||||
|
if measurement.get("discharge_percent")
|
||||||
|
else None
|
||||||
|
),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
points.append(point)
|
points.append(point)
|
||||||
@@ -551,13 +555,13 @@ class SQLAdapter(DatabaseAdapter):
|
|||||||
"station_code": row[1],
|
"station_code": row[1],
|
||||||
"station_name_en": row[2],
|
"station_name_en": row[2],
|
||||||
"station_name_th": row[3],
|
"station_name_th": row[3],
|
||||||
"water_level": float(row[4])
|
"water_level": (
|
||||||
if row[4] is not None
|
float(row[4]) if row[4] is not None else None
|
||||||
else None,
|
),
|
||||||
"discharge": float(row[5]) if row[5] is not None else None,
|
"discharge": float(row[5]) if row[5] is not None else None,
|
||||||
"discharge_percent": float(row[6])
|
"discharge_percent": (
|
||||||
if row[6] is not None
|
float(row[6]) if row[6] is not None else None
|
||||||
else None,
|
),
|
||||||
"status": row[7],
|
"status": row[7],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -611,13 +615,13 @@ class SQLAdapter(DatabaseAdapter):
|
|||||||
"station_code": row[1],
|
"station_code": row[1],
|
||||||
"station_name_en": row[2],
|
"station_name_en": row[2],
|
||||||
"station_name_th": row[3],
|
"station_name_th": row[3],
|
||||||
"water_level": float(row[4])
|
"water_level": (
|
||||||
if row[4] is not None
|
float(row[4]) if row[4] is not None else None
|
||||||
else None,
|
),
|
||||||
"discharge": float(row[5]) if row[5] is not None else None,
|
"discharge": float(row[5]) if row[5] is not None else None,
|
||||||
"discharge_percent": float(row[6])
|
"discharge_percent": (
|
||||||
if row[6] is not None
|
float(row[6]) if row[6] is not None else None
|
||||||
else None,
|
),
|
||||||
"status": row[7],
|
"status": row[7],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -666,13 +670,13 @@ class SQLAdapter(DatabaseAdapter):
|
|||||||
"station_id": row[1],
|
"station_id": row[1],
|
||||||
"station_code": row[2] or f"Station_{row[1]}",
|
"station_code": row[2] or f"Station_{row[1]}",
|
||||||
"station_name_th": row[3] or f"Station {row[1]}",
|
"station_name_th": row[3] or f"Station {row[1]}",
|
||||||
"water_level": float(row[4])
|
"water_level": (
|
||||||
if row[4] is not None
|
float(row[4]) if row[4] is not None else None
|
||||||
else None,
|
),
|
||||||
"discharge": float(row[5]) if row[5] is not None else None,
|
"discharge": float(row[5]) if row[5] is not None else None,
|
||||||
"discharge_percent": float(row[6])
|
"discharge_percent": (
|
||||||
if row[6] is not None
|
float(row[6]) if row[6] is not None else None
|
||||||
else None,
|
),
|
||||||
"status": row[7],
|
"status": row[7],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
+3
-3
@@ -125,9 +125,9 @@ class DatabaseHealthCheck(HealthCheck):
|
|||||||
"message": "Database connection OK",
|
"message": "Database connection OK",
|
||||||
"details": {
|
"details": {
|
||||||
"latest_data_count": len(latest_data),
|
"latest_data_count": len(latest_data),
|
||||||
"latest_timestamp": str(latest_data[0].get("timestamp"))
|
"latest_timestamp": (
|
||||||
if latest_data
|
str(latest_data[0].get("timestamp")) if latest_data else None
|
||||||
else None,
|
),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -207,9 +207,9 @@ def fill_from_hii(
|
|||||||
"timestamp": missing["timestamp"],
|
"timestamp": missing["timestamp"],
|
||||||
"station_code": code,
|
"station_code": code,
|
||||||
"water_level": missing["wl_msl"] - offset,
|
"water_level": missing["wl_msl"] - offset,
|
||||||
"discharge": missing["discharge"]
|
"discharge": (
|
||||||
if code in _HII_EXACT_MIRRORS
|
missing["discharge"] if code in _HII_EXACT_MIRRORS else float("nan")
|
||||||
else float("nan"),
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
fills.append(fill)
|
fills.append(fill)
|
||||||
|
|||||||
+6
-6
@@ -320,9 +320,9 @@ def train_station(
|
|||||||
skipped_heads,
|
skipped_heads,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
skipped_heads[
|
skipped_heads[head_key] = (
|
||||||
head_key
|
f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
|
||||||
] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
|
)
|
||||||
heads[head_key] = clf
|
heads[head_key] = clf
|
||||||
|
|
||||||
if not skip_eval:
|
if not skip_eval:
|
||||||
@@ -417,9 +417,9 @@ def train_station(
|
|||||||
if clf is not None:
|
if clf is not None:
|
||||||
skipped_heads.pop(head_key, None)
|
skipped_heads.pop(head_key, None)
|
||||||
else:
|
else:
|
||||||
skipped_heads[
|
skipped_heads[head_key] = (
|
||||||
head_key
|
f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
|
||||||
] = f"only {n_pos} positives in train span (< {MIN_POSITIVES_FOR_CLASSIFIER})"
|
)
|
||||||
final_heads[head_key] = None
|
final_heads[head_key] = None
|
||||||
|
|
||||||
# v4 = + Mae Ngat dam features; v3 = rise + rain; v2 = rise target only
|
# v4 = + Mae Ngat dam features; v3 = rise + rain; v2 = rise target only
|
||||||
|
|||||||
@@ -51,8 +51,7 @@ class PostgresHistory:
|
|||||||
if start >= end:
|
if start >= end:
|
||||||
raise ValueError("start must be before end")
|
raise ValueError("start must be before end")
|
||||||
|
|
||||||
query = text(
|
query = text("""
|
||||||
"""
|
|
||||||
SELECT m.timestamp, s.station_code, m.water_level,
|
SELECT m.timestamp, s.station_code, m.water_level,
|
||||||
m.discharge, m.discharge_percent
|
m.discharge, m.discharge_percent
|
||||||
FROM water_measurements m
|
FROM water_measurements m
|
||||||
@@ -62,8 +61,7 @@ class PostgresHistory:
|
|||||||
AND m.timestamp <= :end_time
|
AND m.timestamp <= :end_time
|
||||||
ORDER BY m.timestamp ASC
|
ORDER BY m.timestamp ASC
|
||||||
LIMIT :limit
|
LIMIT :limit
|
||||||
"""
|
""")
|
||||||
)
|
|
||||||
with self.engine.connect() as connection:
|
with self.engine.connect() as connection:
|
||||||
rows = connection.execute(
|
rows = connection.execute(
|
||||||
query,
|
query,
|
||||||
@@ -91,9 +89,9 @@ class PostgresHistory:
|
|||||||
"station_code": station_code,
|
"station_code": station_code,
|
||||||
"water_level": water_level,
|
"water_level": water_level,
|
||||||
"discharge": discharge,
|
"discharge": discharge,
|
||||||
"discharge_percent": float(row[4])
|
"discharge_percent": (
|
||||||
if row[4] is not None
|
float(row[4]) if row[4] is not None else None
|
||||||
else None,
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|||||||
+5
-3
@@ -173,8 +173,10 @@ class RequestTracker:
|
|||||||
"failed_requests": self.failed_requests,
|
"failed_requests": self.failed_requests,
|
||||||
"success_rate": self.successful_requests / self.total_requests,
|
"success_rate": self.successful_requests / self.total_requests,
|
||||||
"average_response_time": self.total_response_time / self.total_requests,
|
"average_response_time": self.total_response_time / self.total_requests,
|
||||||
"last_request_time": self.last_request_time.isoformat()
|
"last_request_time": (
|
||||||
if self.last_request_time
|
self.last_request_time.isoformat()
|
||||||
else None,
|
if self.last_request_time
|
||||||
|
else None
|
||||||
|
),
|
||||||
"error_breakdown": dict(self.error_count_by_type),
|
"error_breakdown": dict(self.error_count_by_type),
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-9
@@ -323,9 +323,11 @@ class RidReservoirStore:
|
|||||||
if not preserve_cols:
|
if not preserve_cols:
|
||||||
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
return f"INSERT OR REPLACE INTO {table} ({col_list}) VALUES ({params})"
|
||||||
updates = ", ".join(
|
updates = ", ".join(
|
||||||
f"{c} = COALESCE(excluded.{c}, {table}.{c})"
|
(
|
||||||
if c in preserve_cols
|
f"{c} = COALESCE(excluded.{c}, {table}.{c})"
|
||||||
else f"{c} = excluded.{c}"
|
if c in preserve_cols
|
||||||
|
else f"{c} = excluded.{c}"
|
||||||
|
)
|
||||||
for c in value_cols
|
for c in value_cols
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
@@ -334,9 +336,11 @@ class RidReservoirStore:
|
|||||||
)
|
)
|
||||||
if self.db_type == "postgresql":
|
if self.db_type == "postgresql":
|
||||||
updates = ", ".join(
|
updates = ", ".join(
|
||||||
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
(
|
||||||
if c in preserve_cols
|
f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})"
|
||||||
else f"{c} = EXCLUDED.{c}"
|
if c in preserve_cols
|
||||||
|
else f"{c} = EXCLUDED.{c}"
|
||||||
|
)
|
||||||
for c in value_cols
|
for c in value_cols
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
@@ -344,9 +348,11 @@ class RidReservoirStore:
|
|||||||
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
f"ON CONFLICT ({conflict}) DO UPDATE SET {updates}"
|
||||||
)
|
)
|
||||||
updates = ", ".join(
|
updates = ", ".join(
|
||||||
f"{c} = COALESCE(VALUES({c}), {c})"
|
(
|
||||||
if c in preserve_cols
|
f"{c} = COALESCE(VALUES({c}), {c})"
|
||||||
else f"{c} = VALUES({c})"
|
if c in preserve_cols
|
||||||
|
else f"{c} = VALUES({c})"
|
||||||
|
)
|
||||||
for c in value_cols
|
for c in value_cols
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
|
|||||||
Reference in New Issue
Block a user