ci: green pipelines that check what exists; one formatting contract
The Test Suite job failed on every push since the black check was added because the tree had never been formatted, and pre-commit said 120 columns while CI ran black's default 88. pyproject.toml now carries [tool.black] / [tool.isort] (88, black profile) as the single source; pre-commit reads it; `make format` applied it (13 files, whitespace only, 146 insertions / 128 deletions, tests unchanged at 146 passed). ci.yml: lint (black, isort, flake8 hard errors) + pytest. The Docker registry push, VictoriaMetrics integration test, staging/production deploy and Apache-Bench jobs were template scaffolding for hosts and registries that do not exist; production is a systemd unit updated by git pull. Removed rather than left permanently skipped. docs.yml: the "Check markdown links" step curl'd every URL in every .md and failed on localhost examples and the Tailscale IP, and the Sphinx jobs built artifacts nobody read. Replaced by two checks that mean something: relative links/images in README, CONTRIBUTING and docs/ resolve inside the repo, and the FastAPI OpenAPI schema exports with the documented endpoints present (uploaded as an artifact).
This commit is contained in:
+60
-327
@@ -1,342 +1,75 @@
|
||||
name: CI/CD Pipeline - Northern Thailand Ping River Monitor
|
||||
name: CI
|
||||
|
||||
# What this checks, on every push and PR to master:
|
||||
# 1. formatting contract (black + isort, config in pyproject.toml)
|
||||
# 2. flake8 hard-error gate (syntax, undefined names)
|
||||
# 3. the pytest suite (synthetic data, no DB/network; ~1 min)
|
||||
# Docker build / staging / production / perf jobs from the original template
|
||||
# were removed: there is no registry, no staging host, and production is a
|
||||
# systemd unit deployed by `git pull` on the server (docs/FLOOD_FORECASTING.md
|
||||
# section 6, scripts/install.sh). Re-add a job when the thing it deploys exists.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, develop ]
|
||||
branches: [master, develop]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
branches: [master]
|
||||
schedule:
|
||||
# Run tests daily at 2 AM UTC
|
||||
- cron: '0 2 * * *'
|
||||
# daily, catches dependency drift / upstream API changes in the tests
|
||||
- cron: "0 2 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: '3.11'
|
||||
REGISTRY: git.b4l.co.th
|
||||
IMAGE_NAME: b4l/northern-thailand-ping-river-monitor
|
||||
# GitHub token for better rate limits and authentication
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
PYTHON_VERSION: "3.11" # pandas 2.0.3 ships no 3.12 wheels; psycopg2-binary 2.9.9 breaks on 3.13
|
||||
|
||||
jobs:
|
||||
# Test job
|
||||
lint:
|
||||
name: Format & lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: pip
|
||||
cache-dependency-path: requirements-dev.txt
|
||||
|
||||
- name: Install tools
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore black==23.11.0 isort==5.12.0 flake8==6.1.0
|
||||
|
||||
- name: black
|
||||
run: black --check --diff src/ *.py
|
||||
|
||||
- name: isort
|
||||
run: isort --check-only --diff src/ *.py
|
||||
|
||||
- name: flake8 (errors only)
|
||||
run: flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
|
||||
test:
|
||||
name: Test Suite
|
||||
name: Test suite
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.11'] # pandas 2.0.3 ships no 3.12 wheels; widen after upgrading pandas
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore -r requirements-dev.txt
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=100 --statistics
|
||||
|
||||
- name: Type check with mypy (advisory)
|
||||
run: |
|
||||
# 86 pre-existing errors; blocking typing gate deferred until the debt is paid down
|
||||
mypy src/ --ignore-missing-imports || true
|
||||
|
||||
- name: Format check with black
|
||||
run: |
|
||||
black --check src/ *.py
|
||||
|
||||
- name: Import sort check
|
||||
run: |
|
||||
isort --check-only src/ *.py
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
python tests/test_integration.py
|
||||
|
||||
- name: Run station management tests
|
||||
run: |
|
||||
python tests/test_station_management.py
|
||||
|
||||
- name: Test application startup
|
||||
run: |
|
||||
timeout 10s python run.py --test || true
|
||||
|
||||
- name: Security scan with bandit
|
||||
run: |
|
||||
bandit -r src/ -f json -o bandit-report.json || true
|
||||
|
||||
- name: Upload test artifacts
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: test-results-${{ matrix.python-version }}
|
||||
path: |
|
||||
bandit-report.json
|
||||
*.log
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# Code quality job
|
||||
code-quality:
|
||||
name: Code Quality
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements-dev.txt
|
||||
|
||||
- name: Run safety check
|
||||
run: |
|
||||
safety check -r requirements.txt --json --output safety-report.json || true
|
||||
|
||||
- name: Run bandit security scan
|
||||
run: |
|
||||
bandit -r src/ -f json -o bandit-report.json || true
|
||||
|
||||
- name: Upload security reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: security-reports
|
||||
path: |
|
||||
safety-report.json
|
||||
bandit-report.json
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: pip
|
||||
cache-dependency-path: |
|
||||
requirements.txt
|
||||
requirements-dev.txt
|
||||
|
||||
# Build Docker image
|
||||
build:
|
||||
name: Build Docker Image
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix={{branch}}-
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
|
||||
- name: Test Docker image
|
||||
run: |
|
||||
docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} python run.py --test
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
pip install --root-user-action=ignore pytest==7.4.3 pytest-asyncio==0.21.1
|
||||
|
||||
# Integration test with services
|
||||
integration-test:
|
||||
name: Integration Test with Services
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:latest
|
||||
ports:
|
||||
- 8428:8428
|
||||
options: >-
|
||||
--health-cmd "wget --quiet --tries=1 --spider http://localhost:8428/health"
|
||||
--health-interval 30s
|
||||
--health-timeout 10s
|
||||
--health-retries 3
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Wait for VictoriaMetrics
|
||||
run: |
|
||||
timeout 60s bash -c 'until curl -f http://localhost:8428/health; do sleep 2; done'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip --root-user-action=ignore
|
||||
pip install --root-user-action=ignore -r requirements.txt
|
||||
|
||||
- name: Test with VictoriaMetrics
|
||||
env:
|
||||
DB_TYPE: victoriametrics
|
||||
VM_HOST: localhost
|
||||
VM_PORT: 8428
|
||||
run: |
|
||||
python run.py --test
|
||||
|
||||
- name: Start API server
|
||||
env:
|
||||
DB_TYPE: victoriametrics
|
||||
VM_HOST: localhost
|
||||
VM_PORT: 8428
|
||||
run: |
|
||||
python run.py --web-api &
|
||||
sleep 10
|
||||
|
||||
- name: Test API endpoints
|
||||
run: |
|
||||
curl -f http://localhost:8000/health
|
||||
curl -f http://localhost:8000/stations
|
||||
curl -f http://localhost:8000/metrics
|
||||
|
||||
# Deploy to staging (only on develop branch)
|
||||
deploy-staging:
|
||||
name: Deploy to Staging
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, build, integration-test]
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
environment:
|
||||
name: staging
|
||||
url: https://staging.ping-river-monitor.b4l.co.th
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Deploy to staging
|
||||
run: |
|
||||
echo "Deploying to staging environment..."
|
||||
# Add your staging deployment commands here
|
||||
# Example: kubectl, docker-compose, or webhook call
|
||||
|
||||
- name: Health check staging
|
||||
run: |
|
||||
sleep 30
|
||||
curl -f https://staging.ping-river-monitor.b4l.co.th/health
|
||||
|
||||
# Deploy to production (only on main branch, manual approval)
|
||||
deploy-production:
|
||||
name: Deploy to Production
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, build, integration-test]
|
||||
if: github.ref == 'refs/heads/master'
|
||||
environment:
|
||||
name: production
|
||||
url: https://ping-river-monitor.b4l.co.th
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Deploy to production
|
||||
run: |
|
||||
echo "Deploying to production environment..."
|
||||
# Add your production deployment commands here
|
||||
|
||||
- name: Health check production
|
||||
run: |
|
||||
sleep 30
|
||||
curl -f https://ping-river-monitor.b4l.co.th/health
|
||||
|
||||
- name: Notify deployment
|
||||
run: |
|
||||
echo "✅ Production deployment successful!"
|
||||
echo "🌐 URL: https://ping-river-monitor.b4l.co.th"
|
||||
echo "📊 Grafana: https://grafana.ping-river-monitor.b4l.co.th"
|
||||
|
||||
# Performance test (only on main branch)
|
||||
performance-test:
|
||||
name: Performance Test
|
||||
runs-on: ubuntu-latest
|
||||
needs: deploy-production
|
||||
if: github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
- name: Install Apache Bench
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y apache2-utils
|
||||
|
||||
- name: Performance test API endpoints
|
||||
run: |
|
||||
# Test health endpoint
|
||||
ab -n 100 -c 10 https://ping-river-monitor.b4l.co.th/health
|
||||
|
||||
# Test stations endpoint
|
||||
ab -n 50 -c 5 https://ping-river-monitor.b4l.co.th/stations
|
||||
|
||||
# Test metrics endpoint
|
||||
ab -n 50 -c 5 https://ping-river-monitor.b4l.co.th/metrics
|
||||
|
||||
# Cleanup old artifacts
|
||||
cleanup:
|
||||
name: Cleanup
|
||||
runs-on: ubuntu-latest
|
||||
if: always()
|
||||
needs: [test, build, integration-test]
|
||||
|
||||
steps:
|
||||
- name: Clean up old Docker images
|
||||
run: |
|
||||
echo "Cleaning up old Docker images..."
|
||||
# Add cleanup commands for old images/artifacts
|
||||
- name: pytest
|
||||
env:
|
||||
DB_TYPE: sqlite
|
||||
run: pytest -q -p no:cacheprovider
|
||||
|
||||
Reference in New Issue
Block a user