diff --git a/# b/# new file mode 100644 index 0000000..e69de29 diff --git a/$(wc b/$(wc new file mode 100644 index 0000000..e69de29 diff --git a/.env.back b/.env.back deleted file mode 100644 index bf4dcdc..0000000 --- a/.env.back +++ /dev/null @@ -1,87 +0,0 @@ -# Northern Thailand Ping River Monitor Configuration -# Copy this file to .env and customize for your environment - -# Database Configuration -DB_TYPE=postgresql -# Options: sqlite, mysql, postgresql, influxdb, victoriametrics - -# SQLite Configuration (default) -WATER_DB_PATH=water_levels.db - -# VictoriaMetrics Configuration -VM_HOST=localhost -VM_PORT=8428 -VM_URL= - -# InfluxDB Configuration -INFLUX_HOST=localhost -INFLUX_PORT=8086 -INFLUX_DATABASE=ping_river_monitoring -INFLUX_USERNAME= -INFLUX_PASSWORD= - -# PostgreSQL Configuration (Remote Server) -# Option 1: Full connection string (URL encode special characters in password) -#POSTGRES_CONNECTION_STRING=postgresql://username:url_encoded_password@your-postgres-host:5432/water_monitoring - -# Option 2: Individual components (password will be automatically URL encoded) -POSTGRES_HOST=10.0.10.201 -POSTGRES_PORT=5432 -POSTGRES_DB=ping_river -POSTGRES_USER=ping_river -POSTGRES_PASSWORD=3_%m]k:+16"rx?M#`swIA - -# Examples for connection string: -# - Local: postgresql://postgres:password@localhost:5432/water_monitoring -# - Remote: postgresql://user:pass@192.168.1.100:5432/water_monitoring -# - With special chars: postgresql://user:my%3Apass%40word@host:5432/db -# - With SSL: postgresql://user:pass@host:port/db?sslmode=require -# - Connection pooling: postgresql://user:pass@host:port/db?pool_size=20&max_overflow=0 - -# Special character URL encoding: -# : → %3A @ → %40 # → %23 ? → %3F & → %26 / → %2F % → %25 - -# MySQL Configuration -MYSQL_CONNECTION_STRING=mysql://user:password@localhost:3306/ping_river_monitoring - -# API Configuration -API_HOST=0.0.0.0 -API_PORT=8000 -API_WORKERS=1 - -# Data Collection Settings -SCRAPING_INTERVAL_HOURS=1 -REQUEST_TIMEOUT=30 -MAX_RETRIES=3 -RETRY_DELAY_SECONDS=60 - -# Data Retention -DATA_RETENTION_DAYS=365 - -# Logging Configuration -LOG_LEVEL=INFO -LOG_FILE=water_monitor.log - -# Security (for production) -SECRET_KEY=your-secret-key-here -API_KEY=your-api-key-here - -# Monitoring -ENABLE_METRICS=true -ENABLE_HEALTH_CHECKS=true - -# Geographic Settings -TIMEZONE=Asia/Bangkok -DEFAULT_LATITUDE=18.7875 -DEFAULT_LONGITUDE=99.0045 - -# External Services -NOTIFICATION_EMAIL= -SMTP_SERVER= -SMTP_PORT=587 -SMTP_USERNAME= -SMTP_PASSWORD= - -# Development Settings -DEBUG=false -DEVELOPMENT_MODE=false \ No newline at end of file diff --git a/.env.postgres b/.env.postgres deleted file mode 100644 index fee9cda..0000000 --- a/.env.postgres +++ /dev/null @@ -1,2 +0,0 @@ -DB_TYPE=postgresql -POSTGRES_CONNECTION_STRING=postgresql://postgres:password@localhost:5432/water_monitoring diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml index 6cc1717..e2e7d24 100644 --- a/.gitea/workflows/docs.yml +++ b/.gitea/workflows/docs.yml @@ -357,7 +357,6 @@ jobs: echo "- [README.md](../README.md)" >> docs-summary.md echo "- [API Documentation](../docs/)" >> docs-summary.md echo "- [Contributing Guide](../CONTRIBUTING.md)" >> docs-summary.md - echo "- [Deployment Checklist](../DEPLOYMENT_CHECKLIST.md)" >> docs-summary.md cat docs-summary.md diff --git a/.gitignore b/.gitignore index cc14cbf..54632f6 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,22 @@ models/metrics.json # Playwright MCP browser artifacts (screenshots/snapshots from agent sessions) .playwright-mcp/ + +# Agent tooling state (whole dirs; the entries above only covered subpaths) +.claude/ +.claude-flow/ +.swarm/ +# local MCP server wiring, not project config +.mcp.json +# CLAUDE.md is intentionally NOT ignored — track it if you want the agent +# conventions shared with collaborators; it is untracked today. + +# Model evaluation output (regenerate with scripts/evaluate_variants.py) +models/eval_*.json + +# Editor/merge leftovers and stray shell-redirect artifacts. The repo root once +# collected 56 zero-byte files named after fragments of shell commands. +*.orig +*.rej +*.bak +*~ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 93d6e72..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,129 +0,0 @@ -# GitLab CI/CD Pipeline for Northern Thailand Ping River Monitor - -stages: - - test - - build - - deploy - -variables: - PYTHON_VERSION: "3.11" - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - -cache: - paths: - - .cache/pip - - venv/ - -# Test stage -test: - stage: test - image: python:${PYTHON_VERSION}-slim - before_script: - - apt-get update && apt-get install -y build-essential - - python -m venv venv - - source venv/bin/activate - - pip install --upgrade pip - - pip install -r requirements-dev.txt - script: - - python test_integration.py - - python test_station_management.py - - flake8 src/ --max-line-length=100 - - mypy src/ - coverage: '/TOTAL.*\s+(\d+%)$/' - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - paths: - - htmlcov/ - expire_in: 1 week - -# Code quality -code_quality: - stage: test - image: python:${PYTHON_VERSION}-slim - before_script: - - python -m venv venv - - source venv/bin/activate - - pip install black isort flake8 mypy - script: - - black --check src/ *.py - - isort --check-only src/ *.py - - flake8 src/ --max-line-length=100 - - mypy src/ - allow_failure: true - -# Security scan -security_scan: - stage: test - image: python:${PYTHON_VERSION}-slim - before_script: - - pip install safety bandit - script: - - safety check -r requirements.txt - - bandit -r src/ - allow_failure: true - -# Build Docker image -build: - stage: build - image: docker:latest - services: - - docker:dind - before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - script: - - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA . - - docker build -t $CI_REGISTRY_IMAGE:latest . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE:latest - only: - - main - - develop - -# Deploy to staging -deploy_staging: - stage: deploy - image: alpine:latest - before_script: - - apk add --no-cache curl - script: - - echo "Deploying to staging environment" - - curl -X POST "$STAGING_WEBHOOK_URL" -H "Content-Type: application/json" -d '{"image":"'$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA'"}' - environment: - name: staging - url: https://staging.ping-river-monitor.example.com - only: - - develop - -# Deploy to production -deploy_production: - stage: deploy - image: alpine:latest - before_script: - - apk add --no-cache curl - script: - - echo "Deploying to production environment" - - curl -X POST "$PRODUCTION_WEBHOOK_URL" -H "Content-Type: application/json" -d '{"image":"'$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA'"}' - environment: - name: production - url: https://ping-river-monitor.example.com - when: manual - only: - - main - -# Health check after deployment -health_check: - stage: deploy - image: alpine:latest - before_script: - - apk add --no-cache curl jq - script: - - sleep 30 # Wait for deployment - - curl -f $HEALTH_CHECK_URL/health - - curl -s $HEALTH_CHECK_URL/metrics | jq . - dependencies: - - deploy_production - only: - - main \ No newline at end of file diff --git a/1.24 b/1.24 new file mode 100644 index 0000000..e69de29 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/DEBUG b/DEBUG new file mode 100644 index 0000000..e69de29 diff --git a/DEPLOYMENT_CHECKLIST.md b/DEPLOYMENT_CHECKLIST.md deleted file mode 100644 index d9f94ae..0000000 --- a/DEPLOYMENT_CHECKLIST.md +++ /dev/null @@ -1,268 +0,0 @@ -# 🚀 Deployment Checklist - Northern Thailand Ping River Monitor - -## ✅ Pre-Deployment Checklist - -### **Code Quality** -- [ ] All tests pass (`make test`) -- [ ] Code formatting applied (`make format`) -- [ ] Linting checks pass (`make lint`) -- [ ] No security vulnerabilities (`safety check`) -- [ ] Documentation updated -- [ ] Version number updated in `setup.py` and `src/__init__.py` - -### **Configuration** -- [ ] Environment variables configured (`.env` file) -- [ ] Database connection tested -- [ ] API endpoints tested -- [ ] Log levels appropriate for environment -- [ ] Security settings configured (API keys, secrets) -- [ ] Resource limits set (memory, CPU) - -### **Dependencies** -- [ ] All required packages in `requirements.txt` -- [ ] No unused dependencies -- [ ] Security updates applied -- [ ] Compatible Python version (3.9+) - -## 🐳 Docker Deployment - -### **Pre-Docker Checklist** -- [ ] Dockerfile tested locally -- [ ] Docker Compose configuration verified -- [ ] Volume mounts configured correctly -- [ ] Network settings configured -- [ ] Health checks working -- [ ] Resource limits set - -### **Docker Commands** -```bash -# Build and test locally -make docker-build -docker run --rm ping-river-monitor python run.py --test - -# Deploy with Docker Compose -make docker-run - -# Verify deployment -make health-check -``` - -### **Post-Docker Checklist** -- [ ] All services running (`docker-compose ps`) -- [ ] Health checks passing -- [ ] Logs showing normal operation -- [ ] API accessible (`curl http://localhost:8000/health`) -- [ ] Database connectivity verified -- [ ] Grafana dashboards loading - -## 🌐 Production Deployment - -### **Infrastructure Requirements** -- [ ] Server specifications adequate (CPU, RAM, Storage) -- [ ] Network connectivity to external APIs -- [ ] SSL certificates configured (if HTTPS) -- [ ] Firewall rules configured -- [ ] Backup strategy implemented -- [ ] Monitoring alerts configured - -### **Security Checklist** -- [ ] API keys secured (environment variables) -- [ ] Database credentials secured -- [ ] HTTPS enabled for web interface -- [ ] Input validation enabled -- [ ] Rate limiting configured -- [ ] Log sanitization enabled - -### **Performance Checklist** -- [ ] Database indexes created -- [ ] Connection pooling configured -- [ ] Caching enabled where appropriate -- [ ] Resource monitoring enabled -- [ ] Performance baselines established - -## 📊 Monitoring Setup - -### **Health Monitoring** -- [ ] Health check endpoints responding -- [ ] Database health monitoring -- [ ] API response time monitoring -- [ ] Memory usage monitoring -- [ ] Disk space monitoring - -### **Alerting** -- [ ] Critical error alerts configured -- [ ] Performance degradation alerts -- [ ] Database connectivity alerts -- [ ] Disk space alerts -- [ ] API availability alerts - -### **Logging** -- [ ] Log rotation configured -- [ ] Log levels appropriate -- [ ] Structured logging enabled -- [ ] Log aggregation configured (if applicable) -- [ ] Log retention policy set - -## 🔄 CI/CD Pipeline - -### **GitLab CI/CD** -- [ ] `.gitlab-ci.yml` configured -- [ ] Pipeline variables set -- [ ] Test stage passing -- [ ] Build stage creating artifacts -- [ ] Deploy stage configured -- [ ] Rollback procedure documented - -### **Pipeline Stages** -- [ ] **Test**: Unit tests, integration tests, linting -- [ ] **Build**: Docker image creation, artifact generation -- [ ] **Deploy**: Staging deployment, production deployment -- [ ] **Verify**: Health checks, smoke tests - -## 🗄️ Database Setup - -### **Database Configuration** -- [ ] Database server running and accessible -- [ ] Database created with correct permissions -- [ ] Connection string configured -- [ ] Migration scripts run (if applicable) -- [ ] Backup strategy implemented -- [ ] Performance tuning applied - -### **Database-Specific Checklist** - -#### **SQLite** -- [ ] Database file permissions set correctly -- [ ] WAL mode enabled for better concurrency -- [ ] Regular backup scheduled - -#### **MySQL/PostgreSQL** -- [ ] User accounts created with minimal privileges -- [ ] Connection pooling configured -- [ ] Query performance optimized -- [ ] Replication configured (if applicable) - -#### **InfluxDB** -- [ ] Retention policies configured -- [ ] Continuous queries set up (if needed) -- [ ] Backup strategy implemented - -#### **VictoriaMetrics** -- [ ] Storage configuration optimized -- [ ] Retention period set -- [ ] Resource limits configured - -## 🌐 Web Interface - -### **API Deployment** -- [ ] FastAPI server running -- [ ] All endpoints responding correctly -- [ ] API documentation accessible (`/docs`) -- [ ] CORS configured correctly -- [ ] Rate limiting working -- [ ] Authentication configured (if applicable) - -### **Frontend Integration** -- [ ] Grafana dashboards configured -- [ ] Data sources connected -- [ ] Visualizations working -- [ ] Alerts configured -- [ ] User access configured - -## 📈 Performance Verification - -### **Load Testing** -- [ ] API endpoints tested under load -- [ ] Database performance under load -- [ ] Memory usage under load -- [ ] Response times acceptable -- [ ] Error rates acceptable - -### **Capacity Planning** -- [ ] Expected data volume calculated -- [ ] Storage growth projected -- [ ] Scaling strategy documented -- [ ] Resource monitoring thresholds set - -## 🔧 Operational Procedures - -### **Maintenance** -- [ ] Update procedure documented -- [ ] Backup and restore procedures tested -- [ ] Rollback procedure documented -- [ ] Monitoring runbooks created -- [ ] Incident response procedures documented - -### **Documentation** -- [ ] Deployment guide updated -- [ ] API documentation current -- [ ] Configuration documentation complete -- [ ] Troubleshooting guide available -- [ ] Contact information updated - -## ✅ Post-Deployment Verification - -### **Functional Testing** -- [ ] Data collection working -- [ ] API endpoints responding -- [ ] Database writes successful -- [ ] Web interface accessible -- [ ] Station management working - -### **Integration Testing** -- [ ] External API connectivity -- [ ] Database integration -- [ ] Monitoring integration -- [ ] Alert system working -- [ ] Backup system working - -### **Performance Testing** -- [ ] Response times acceptable -- [ ] Memory usage normal -- [ ] CPU usage normal -- [ ] Disk I/O normal -- [ ] Network usage normal - -## 🚨 Rollback Plan - -### **Rollback Triggers** -- [ ] Critical errors in production -- [ ] Performance degradation -- [ ] Data corruption -- [ ] Security vulnerabilities -- [ ] Service unavailability - -### **Rollback Procedure** -1. [ ] Stop current deployment -2. [ ] Restore previous Docker images -3. [ ] Restore database backup (if needed) -4. [ ] Verify system functionality -5. [ ] Update monitoring and alerts -6. [ ] Document incident and lessons learned - -## 📞 Support Information - -### **Emergency Contacts** -- [ ] System administrator contact -- [ ] Database administrator contact -- [ ] Network administrator contact -- [ ] Application developer contact - -### **Documentation Links** -- [ ] Deployment guide -- [ ] API documentation -- [ ] Troubleshooting guide -- [ ] Configuration reference -- [ ] Monitoring dashboards - ---- - -**Deployment Date**: ___________ -**Deployed By**: ___________ -**Version**: v3.1.3 -**Environment**: ___________ - -**Sign-off**: -- [ ] Technical Lead: ___________ -- [ ] Operations Team: ___________ -- [ ] Security Team: ___________ \ No newline at end of file diff --git a/FINAL_CHECKLIST.md b/FINAL_CHECKLIST.md deleted file mode 100644 index 7b15211..0000000 --- a/FINAL_CHECKLIST.md +++ /dev/null @@ -1,193 +0,0 @@ -# Final GitHub Publication Checklist ✅ - -This checklist ensures the Thailand Water Level Monitor project is ready for GitHub publication. - -## 🎯 **Project Preparation Complete** - -### ✅ **Core Repository Files** -- [x] **README.md** - Comprehensive project documentation with badges and quick start -- [x] **LICENSE** - MIT License for open source distribution -- [x] **CONTRIBUTING.md** - Detailed contributor guidelines -- [x] **.gitignore** - Comprehensive ignore rules for all file types -- [x] **requirements.txt** - All Python dependencies listed and tested - -### ✅ **Source Code Organization** -- [x] **src/** directory created with clean separation -- [x] **scripts/** directory for utility scripts and system files -- [x] **docs/** directory with comprehensive documentation -- [x] **grafana/** directory with visualization configuration -- [x] All temporary files removed (*.db, *.log, __pycache__) - -### ✅ **Documentation Quality** -- [x] **Installation guides** for all platforms and databases -- [x] **Configuration examples** for 5 different database types -- [x] **Troubleshooting guides** for common deployment issues -- [x] **Migration guides** for updating existing systems -- [x] **API references** documenting Thai government data sources -- [x] **Notable documents** section with official resources - -### ✅ **Production Readiness** -- [x] **Docker support** with Dockerfile and docker-compose -- [x] **Systemd service** configuration for Linux deployment -- [x] **Multi-database support** (SQLite, PostgreSQL, MySQL, InfluxDB, VictoriaMetrics) -- [x] **Geolocation support** for Grafana geomap visualization -- [x] **Migration scripts** for safe database schema updates -- [x] **HTTPS configuration** guide for secure deployment - -### ✅ **Code Quality** -- [x] **Modular architecture** with clean separation of concerns -- [x] **Error handling** and comprehensive logging -- [x] **Configuration management** via environment variables -- [x] **Database abstraction** layer for multiple backends -- [x] **Testing utilities** (demo_databases.py) - -### ✅ **Features Verified** -- [x] **Real-time data collection** from 16 Thai water stations -- [x] **15-minute scheduling** with intelligent retry logic -- [x] **Gap filling** for missing historical data -- [x] **Data validation** and error recovery -- [x] **Geolocation integration** with sample coordinates -- [x] **Grafana dashboards** with pre-built visualizations - -## 🚀 **Ready for GitHub Publication** - -### **Repository Information** -- **Name**: `thailand-water-monitor` -- **Description**: "Real-time water level monitoring system for Thailand's Royal Irrigation Department stations with Grafana visualization" -- **Topics**: `water-monitoring`, `thailand`, `grafana`, `timeseries`, `python`, `iot`, `environmental-monitoring` -- **License**: MIT -- **Language**: Python - -### **Repository Settings** -- [x] Enable Issues for bug reports and feature requests -- [x] Enable Discussions for community support -- [x] Enable Wiki for extended documentation -- [x] Set up GitHub Pages for documentation hosting -- [x] Configure branch protection for main branch - -### **Initial Release (v1.0.0)** -- **Release Title**: "Thailand Water Level Monitor v1.0.0 - Complete Monitoring Solution" -- **Release Notes**: - - Complete real-time monitoring system - - Multi-database backend support - - Grafana geomap integration - - Production-ready deployment - - Comprehensive documentation - -## 📊 **Project Statistics** - -### **Code Metrics** -- **Total Files**: 25+ files -- **Python Source Files**: 4 main modules -- **Documentation Files**: 12 comprehensive guides -- **Configuration Files**: 6 deployment configurations -- **Lines of Code**: ~2,000+ lines of Python -- **Documentation**: ~15,000+ words - -### **Feature Coverage** -- **Database Backends**: 5 different types supported -- **Monitoring Stations**: 16 across Thailand -- **Data Collection**: Every 15 minutes -- **Data Points**: ~300 measurements per collection cycle -- **Geolocation**: GPS coordinates and geohash support -- **Visualization**: Pre-built Grafana dashboards - -### **Documentation Coverage** -- **Installation**: Complete setup for all platforms -- **Configuration**: All database types documented -- **Deployment**: Docker, systemd, manual options -- **Troubleshooting**: Common issues and solutions -- **Migration**: Safe upgrade procedures -- **API**: External data source documentation - -## 🌟 **Key Selling Points** - -### **For Water Management Professionals** -- Real-time monitoring of 16 stations across Thailand -- Historical data analysis and trend visualization -- Alert capabilities for critical water levels -- Integration with official Thai government data sources - -### **For Developers** -- Clean, modular Python codebase -- Multiple database backend options -- Docker containerization for easy deployment -- Comprehensive API documentation - -### **For System Administrators** -- Production-ready deployment configurations -- Systemd service integration -- HTTPS and security configuration -- Monitoring and logging capabilities - -### **For Data Scientists** -- Time-series data with geolocation -- Grafana visualization and analysis tools -- Historical data gap filling -- Export capabilities for further analysis - -## 🎯 **Post-Publication Roadmap** - -### **Immediate (Week 1)** -- [ ] Create GitHub repository and upload files -- [ ] Set up initial release v1.0.0 -- [ ] Configure repository settings and templates -- [ ] Create project documentation website - -### **Short-term (Month 1)** -- [ ] Add GitHub Actions for CI/CD -- [ ] Create issue and PR templates -- [ ] Set up automated testing -- [ ] Add code quality badges - -### **Medium-term (Quarter 1)** -- [ ] Community feedback integration -- [ ] Additional database backends -- [ ] Mobile app development -- [ ] Advanced alerting system - -### **Long-term (Year 1)** -- [ ] Predictive analytics features -- [ ] Machine learning integration -- [ ] Multi-country expansion -- [ ] Commercial support options - -## 🏆 **Success Metrics** - -### **Community Engagement** -- GitHub stars and forks -- Issue reports and feature requests -- Community contributions -- Documentation feedback - -### **Technical Adoption** -- Download and deployment statistics -- Database backend usage patterns -- Performance benchmarks -- User success stories - -### **Impact Measurement** -- Water management improvements -- Early warning system effectiveness -- Data accessibility improvements -- Research and academic usage - ---- - -## ✅ **FINAL VERIFICATION** - -**All checklist items completed successfully!** - -The Thailand Water Level Monitor project is now: -- ✅ **Professionally organized** with clean structure -- ✅ **Comprehensively documented** with guides for all use cases -- ✅ **Production ready** with multiple deployment options -- ✅ **Community friendly** with contribution guidelines -- ✅ **Feature complete** with real-time monitoring capabilities - -**🚀 Ready for GitHub publication and community engagement!** 🌊 - ---- - -*Last updated: July 30, 2025* -*Project status: Ready for publication* diff --git a/GITEA_SETUP_SUMMARY.md b/GITEA_SETUP_SUMMARY.md deleted file mode 100644 index b5cf9c5..0000000 --- a/GITEA_SETUP_SUMMARY.md +++ /dev/null @@ -1,233 +0,0 @@ -# 🎉 Gitea Actions Setup Complete! - -## 🚀 **What's Been Created** - -Your **Northern Thailand Ping River Monitor** now has a complete CI/CD pipeline with Gitea Actions! Here's what's been set up: - -### **🔄 Gitea Actions Workflows** - -``` -.gitea/workflows/ -├── ci.yml # Main CI/CD pipeline -├── release.yml # Automated releases -├── security.yml # Security & dependency scanning -└── docs.yml # Documentation generation -``` - -### **📊 Workflow Features** - -#### **1. CI/CD Pipeline (`ci.yml`)** -- ✅ **Multi-Python Testing** (3.9, 3.10, 3.11, 3.12) -- ✅ **Code Quality Checks** (flake8, mypy, black, isort) -- ✅ **Docker Multi-Arch Builds** (amd64, arm64) -- ✅ **Integration Testing** with VictoriaMetrics -- ✅ **Automated Staging Deployment** (develop branch) -- ✅ **Manual Production Deployment** (main branch) -- ✅ **Performance Testing** after deployment - -#### **2. Release Management (`release.yml`)** -- 🏷️ **Tag-Based Releases** (`v*.*.*` pattern) -- 📝 **Automatic Changelog Generation** -- 🐳 **Multi-Architecture Docker Images** -- 🔒 **Security Scanning** before release -- ✅ **Comprehensive Validation** after deployment - -#### **3. Security Monitoring (`security.yml`)** -- 🔒 **Daily Security Scans** (3 AM UTC) -- 📦 **Dependency Vulnerability Detection** -- 🐳 **Docker Image Security Scanning** -- 📄 **License Compliance Checking** -- 📊 **Code Quality Metrics** -- 🔄 **Automated Update Notifications** - -#### **4. Documentation (`docs.yml`)** -- 📚 **API Documentation Generation** -- 🔗 **Link Validation** -- 📖 **Sphinx Documentation Building** -- ✅ **Documentation Completeness Checking** - -## 🔧 **Setup Instructions** - -### **1. Configure Repository Secrets** - -In your Gitea repository settings, add these secrets: - -```bash -# Required -GITEA_TOKEN # For container registry access - -# Optional (for notifications) -SLACK_WEBHOOK_URL # Slack notifications -STAGING_WEBHOOK_URL # Staging deployment webhook -PRODUCTION_WEBHOOK_URL # Production deployment webhook -``` - -### **2. Enable Actions** - -1. Go to your repository settings in Gitea -2. Enable "Actions" if not already enabled -3. Configure runners if using self-hosted runners - -### **3. Push to Repository** - -```bash -# Initialize and push -git init -git remote add origin https://git.b4l.co.th/grabowski/Northern-Thailand-Ping-River-Monitor.git -git add . -git commit -m "Initial commit with Gitea Actions workflows" -git push -u origin main -``` - -## 🎯 **Workflow Triggers** - -### **Automatic Triggers** -- **Push to main/develop** → CI/CD Pipeline -- **Pull Request to main** → Testing & Validation -- **Daily at 2 AM UTC** → CI/CD Health Check -- **Daily at 3 AM UTC** → Security Scanning -- **Git Tag `v*.*.*`** → Release Pipeline -- **Documentation Changes** → Documentation Build - -### **Manual Triggers** -- **Manual Dispatch** → Any workflow can be triggered manually -- **Release Creation** → Manual release with custom version - -## 📊 **Monitoring & Status** - -### **Status Badges** -Your README now includes comprehensive status badges: -- CI/CD Pipeline Status -- Security Scan Status -- Documentation Build Status -- Python Version Support -- FastAPI Version -- Docker Ready -- License Information -- Current Version - -### **Workflow Artifacts** -Each workflow generates useful artifacts: -- **Test Results** and coverage reports -- **Security Scan Reports** (JSON format) -- **Docker Images** (multi-architecture) -- **Documentation** (HTML and PDF) -- **Performance Reports** - -## 🚀 **Usage Examples** - -### **Development Workflow** -```bash -# Create feature branch -git checkout -b feature/new-station-type -# Make changes -git add . -git commit -m "Add support for new station type" -git push origin feature/new-station-type -# Create PR in Gitea → Triggers testing -``` - -### **Release Workflow** -```bash -# Create and push release tag -git tag v3.1.1 -git push origin v3.1.1 -# → Triggers automated release pipeline -``` - -### **Security Monitoring** -- **Daily scans** run automatically -- **Security reports** available in Actions artifacts -- **Notifications** sent for critical vulnerabilities - -## 🔍 **Validation Commands** - -Test your setup locally: - -```bash -# Validate workflow syntax -make validate-workflows - -# Test workflow components -make workflow-test - -# Run full test suite -make test - -# Build Docker image -make docker-build -``` - -## 📈 **Performance & Optimization** - -### **Caching Strategy** -- **Pip dependencies** cached across runs -- **Docker layers** cached for faster builds -- **Workflow artifacts** retained for analysis - -### **Parallel Execution** -- **Matrix builds** for multiple Python versions -- **Independent jobs** for security and testing -- **Conditional execution** to skip unnecessary steps - -### **Resource Management** -- **Appropriate timeouts** prevent hanging workflows -- **Artifact cleanup** manages storage usage -- **Efficient Docker builds** with multi-stage approach - -## 🔒 **Security Best Practices** - -### **Implemented Security** -- ✅ **Secret management** via Gitea repository secrets -- ✅ **Multi-stage Docker builds** for minimal attack surface -- ✅ **Non-root containers** for better security -- ✅ **Vulnerability scanning** before deployment -- ✅ **Dependency monitoring** with automated alerts - -### **Security Scanning Coverage** -- **Python dependencies** (Safety, Bandit) -- **Docker images** (Trivy) -- **Code quality** (Semgrep) -- **License compliance** (pip-licenses) - -## 📚 **Documentation** - -### **Available Documentation** -- [Gitea Workflows Guide](docs/GITEA_WORKFLOWS.md) - Detailed workflow documentation -- [Contributing Guide](CONTRIBUTING.md) - How to contribute -- [Deployment Checklist](DEPLOYMENT_CHECKLIST.md) - Production deployment -- [Project Structure](docs/PROJECT_STRUCTURE.md) - Architecture overview - -### **Generated Documentation** -- **API Documentation** - Auto-generated from OpenAPI spec -- **Code Documentation** - Sphinx-generated from docstrings -- **Security Reports** - Automated vulnerability reports - -## 🎉 **Ready for Production!** - -Your repository is now equipped with: - -- 🔄 **Enterprise-grade CI/CD pipeline** -- 🔒 **Comprehensive security monitoring** -- 📊 **Automated quality assurance** -- 🚀 **Streamlined release management** -- 📚 **Automated documentation** -- 🐳 **Multi-architecture Docker support** -- 📈 **Performance monitoring** -- 🔍 **Comprehensive testing** - -## 🚀 **Next Steps** - -1. **Push to Gitea** and watch the workflows run -2. **Configure deployment environments** (staging/production) -3. **Set up monitoring dashboards** for workflow metrics -4. **Configure notifications** for team collaboration -5. **Create your first release** with `git tag v3.1.3` - -Your **Northern Thailand Ping River Monitor** is now ready for professional development and deployment! 🎊 - ---- - -**Workflow Version**: v3.1.3 -**Setup Date**: 2025-08-12 -**Repository**: https://git.b4l.co.th/grabowski/Northern-Thailand-Ping-River-Monitor \ No newline at end of file diff --git a/GITHUB_PUBLICATION_SUMMARY.md b/GITHUB_PUBLICATION_SUMMARY.md deleted file mode 100644 index d4b6721..0000000 --- a/GITHUB_PUBLICATION_SUMMARY.md +++ /dev/null @@ -1,203 +0,0 @@ -# GitHub Publication Summary - -This document summarizes the Thailand Water Level Monitor project preparation for GitHub publication. - -## 📁 **Final Project Structure** - -``` -thailand-water-monitor/ -├── 📄 README.md # Main project documentation -├── 📄 LICENSE # MIT License -├── 📄 CONTRIBUTING.md # Contributor guidelines -├── 📄 requirements.txt # Python dependencies -├── 📄 .gitignore # Git ignore rules -├── 📄 Dockerfile # Container definition -├── 📄 docker-compose.victoriametrics.yml # Complete stack deployment -│ -├── 📂 src/ # Source Code -│ ├── 🐍 water_scraper_v3.py # Main application -│ ├── 🐍 database_adapters.py # Multi-database support -│ ├── 🐍 config.py # Configuration management -│ └── 🐍 demo_databases.py # Database testing utility -│ -├── 📂 scripts/ # Utility Scripts -│ ├── 🐍 migrate_geolocation.py # Database migration script -│ └── ⚙️ water-monitor.service # Systemd service file -│ -├── 📂 docs/ # Documentation -│ ├── 📖 DATABASE_DEPLOYMENT_GUIDE.md # Complete setup guide -│ ├── 📖 ENHANCED_SCHEDULER_GUIDE.md # 15-minute scheduling -│ ├── 📖 GEOLOCATION_GUIDE.md # Grafana geomap integration -│ ├── 📖 GAP_FILLING_GUIDE.md # Data integrity management -│ ├── 📖 MIGRATION_QUICKSTART.md # Quick migration guide -│ ├── 📖 VICTORIAMETRICS_SETUP.md # High-performance deployment -│ ├── 📖 HTTPS_CONFIGURATION.md # Secure deployment -│ ├── 📖 DEBIAN_TROUBLESHOOTING.md # Linux deployment issues -│ ├── 📖 PROJECT_STATUS.md # Development status -│ └── 📂 references/ -│ └── 📖 NOTABLE_DOCUMENTS.md # Official Thai government resources -│ -└── 📂 grafana/ # Grafana Configuration - ├── 📂 dashboards/ - │ └── 📊 water-monitoring-dashboard.json - └── 📂 provisioning/ - ├── 📂 dashboards/ - │ └── ⚙️ dashboard.yml - └── 📂 datasources/ - └── ⚙️ victoriametrics.yml -``` - -## ✅ **GitHub Readiness Checklist** - -### **Core Files** -- ✅ **README.md** - Comprehensive project documentation with badges, features, quick start -- ✅ **LICENSE** - MIT License for open source distribution -- ✅ **CONTRIBUTING.md** - Detailed contributor guidelines and development setup -- ✅ **.gitignore** - Comprehensive ignore rules for Python, databases, logs, IDE files -- ✅ **requirements.txt** - All Python dependencies listed - -### **Source Code Organization** -- ✅ **src/** directory - Clean separation of source code -- ✅ **scripts/** directory - Utility scripts and system files -- ✅ **docs/** directory - Comprehensive documentation -- ✅ **grafana/** directory - Visualization configuration - -### **Documentation Quality** -- ✅ **Installation guides** - Multiple deployment options -- ✅ **Configuration examples** - All database types covered -- ✅ **Troubleshooting guides** - Common issues and solutions -- ✅ **Migration guides** - Updating existing systems -- ✅ **API references** - External data sources documented - -### **Production Readiness** -- ✅ **Docker support** - Containerization ready -- ✅ **Systemd service** - Linux service configuration -- ✅ **Multi-database support** - 5 different database options -- ✅ **Geolocation support** - Grafana geomap integration -- ✅ **Migration scripts** - Safe database updates - -## 🌟 **Key Features for GitHub** - -### **Real-time Monitoring** -- 16 water stations across Thailand -- 15-minute data collection frequency -- Automatic gap filling and data validation -- Multi-database backend support - -### **Visualization Ready** -- Pre-built Grafana dashboards -- Geomap integration with coordinates -- Real-time alerts and notifications -- Historical trend analysis - -### **Production Deployment** -- Docker containerization -- VictoriaMetrics high-performance backend -- HTTPS and security configuration -- Comprehensive logging and monitoring - -### **Developer Friendly** -- Clean, modular code structure -- Comprehensive documentation -- Multiple database adapters -- Easy local development setup - -## 📊 **Project Statistics** - -### **Code Metrics** -- **Python Files**: 4 main source files -- **Documentation**: 10+ comprehensive guides -- **Database Support**: 5 different backends -- **Monitoring Stations**: 16 across Thailand -- **Data Points**: ~300 every 15 minutes - -### **Documentation Coverage** -- **Installation**: Complete setup guides for all platforms -- **Configuration**: All database types documented -- **Deployment**: Docker, systemd, and manual options -- **Troubleshooting**: Common issues and solutions -- **Migration**: Safe upgrade procedures - -### **Features Implemented** -- ✅ Real-time data collection -- ✅ Multi-database support -- ✅ Geolocation integration -- ✅ Gap filling and data validation -- ✅ Grafana visualization -- ✅ Docker deployment -- ✅ Production monitoring -- ✅ Migration tools - -## 🚀 **Ready for GitHub Publication** - -### **Repository Setup** -1. **Create GitHub repository** - "thailand-water-monitor" -2. **Upload all files** - Complete project structure -3. **Configure repository settings**: - - Add description: "Real-time water level monitoring for Thailand's RID stations" - - Add topics: `water-monitoring`, `thailand`, `grafana`, `timeseries`, `python` - - Enable Issues and Discussions - - Set up GitHub Pages for documentation - -### **Initial Release** -- **Version**: v1.0.0 -- **Release Notes**: Complete feature set with multi-database support -- **Assets**: Include sample configuration files -- **Documentation**: Link to comprehensive guides - -### **Community Features** -- **Issues Template**: Bug reports and feature requests -- **Pull Request Template**: Contribution guidelines -- **Discussions**: Community support and questions -- **Wiki**: Extended documentation and tutorials - -## 🎯 **Post-Publication Tasks** - -### **Community Building** -- Create detailed issue templates -- Set up GitHub Actions for CI/CD -- Add code quality badges -- Create project roadmap - -### **Documentation Enhancement** -- Add video tutorials -- Create API documentation -- Add performance benchmarks -- Create deployment examples - -### **Feature Development** -- Mobile app integration -- Additional database backends -- Advanced alerting system -- Predictive analytics - -## 📞 **Support Channels** - -- **GitHub Issues**: Bug reports and feature requests -- **GitHub Discussions**: Community support and questions -- **Documentation**: Comprehensive guides in docs/ directory -- **Examples**: Working configurations and deployments - -## 🏆 **Project Highlights** - -### **Technical Excellence** -- Clean, modular architecture -- Comprehensive error handling -- Production-ready deployment -- Multi-database abstraction - -### **Documentation Quality** -- Step-by-step installation guides -- Troubleshooting for common issues -- Migration procedures for updates -- API and configuration references - -### **Community Ready** -- Open source MIT license -- Contributor guidelines -- Development setup instructions -- Code quality standards - ---- - -**The Thailand Water Level Monitor project is now fully prepared for GitHub publication with a professional structure, comprehensive documentation, and production-ready features.** 🌊 diff --git a/GITHUB_TOKEN_SETUP.md b/GITHUB_TOKEN_SETUP.md deleted file mode 100644 index 132b394..0000000 --- a/GITHUB_TOKEN_SETUP.md +++ /dev/null @@ -1,114 +0,0 @@ -# 🔑 GitHub Token Setup Guide - -## 🎯 **Why You Need This** - -The Gitea Actions workflows use Trivy for security scanning, which needs to download vulnerability databases from GitHub. Without a GitHub token, you'll hit rate limits and the security scans will fail. - -## 🚀 **Quick Setup (5 minutes)** - -### **Step 1: Create GitHub Personal Access Token** - -1. **Go to GitHub**: https://github.com/settings/tokens -2. **Click "Generate new token"** → "Generate new token (classic)" -3. **Configure the token**: - - **Note**: `B4L Ping River Monitor - Gitea Actions` - - **Expiration**: `90 days` (or longer) - - **Scopes**: Select `public_repo` (for public repositories) -4. **Click "Generate token"** -5. **Copy the token** (you won't see it again!) - -### **Step 2: Add Token to Gitea Repository** - -1. **Go to your repository**: https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor -2. **Click "Settings"** (in the repository) -3. **Click "Secrets"** in the left sidebar -4. **Click "Add Secret"** -5. **Configure the secret**: - - **Name**: `GITHUB_TOKEN` - - **Value**: Paste the token you copied from GitHub -6. **Click "Add Secret"** - -### **Step 3: Verify It's Working** - -1. **Trigger a workflow** by pushing a commit or manually running the security workflow -2. **Check the Actions tab** in your repository -3. **Look for the message**: `✅ GITHUB_TOKEN is configured` - -## 🔒 **Security Best Practices** - -### **Token Permissions** -- **Minimum required**: `public_repo` scope -- **Never use**: `repo` scope unless you need private repo access -- **Avoid**: Admin or write permissions - -### **Token Management** -- **Set expiration**: Don't create tokens that never expire -- **Regular rotation**: Update tokens every 90 days -- **Monitor usage**: Check GitHub token usage in settings - -### **Repository Security** -- **Only trusted contributors**: Should have access to repository secrets -- **Audit regularly**: Review who has access to secrets -- **Use organization secrets**: For multiple repositories - -## 🧪 **Testing the Setup** - -### **Manual Test** -```bash -# Trigger the security workflow manually -# Go to: Repository → Actions → Security & Dependency Updates → Run workflow -``` - -### **Automatic Test** -```bash -# Push any change to trigger workflows -git commit --allow-empty -m "Test GitHub token setup" -git push -``` - -### **Check Workflow Logs** -1. Go to Actions tab in your repository -2. Click on the latest "Security & Dependency Updates" run -3. Click on "Docker Security Scan" job -4. Look for: `✅ GITHUB_TOKEN is configured` - -## ❌ **Troubleshooting** - -### **"GITHUB_TOKEN not configured" message** -- **Problem**: Token not added to repository secrets -- **Solution**: Follow Step 2 above, ensure exact name `GITHUB_TOKEN` - -### **"Bad credentials" error** -- **Problem**: Token is invalid or expired -- **Solution**: Generate a new token and update the secret - -### **Rate limit errors** -- **Problem**: Token doesn't have correct permissions -- **Solution**: Ensure token has `public_repo` scope - -### **Trivy still failing** -- **Problem**: Network issues or GitHub API problems -- **Solution**: Wait and retry, or check GitHub status page - -## 🎉 **Success Indicators** - -When everything is working correctly, you'll see: - -✅ **In workflow logs**: `✅ GITHUB_TOKEN is configured` -✅ **Security scans**: Complete without authentication errors -✅ **Trivy reports**: Generated and uploaded as artifacts -✅ **No rate limit errors**: In the workflow execution - -## 📚 **Additional Resources** - -- [GitHub Personal Access Tokens Documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) -- [Gitea Secrets Documentation](https://docs.gitea.io/en-us/usage/actions/#secrets) -- [Trivy Action Documentation](https://github.com/aquasecurity/trivy-action) - ---- - -**Setup Time**: ~5 minutes -**Token Validity**: 90 days (recommended) -**Security Level**: High (read-only public repo access) - -Your workflows will now run smoothly with proper GitHub API authentication! 🚀 \ No newline at end of file diff --git a/Makefile b/Makefile index cffb11d..ee6b4a4 100644 --- a/Makefile +++ b/Makefile @@ -120,10 +120,6 @@ docker-logs: docs: cd docs && make html -# Database management -db-migrate: - uv run python scripts/migrate_geolocation.py - # Monitoring health-check: curl -f http://localhost:8000/health || exit 1 @@ -149,9 +145,6 @@ setup-postgres: test-postgres: uv run python -c "from scripts.setup_postgres import test_postgres_connection; from src.config import Config; config = Config.get_database_config(); test_postgres_connection(config['connection_string'])" -encode-password: - uv run python scripts/encode_password.py - migrate-sqlite: uv run python scripts/migrate_sqlite_to_postgres.py @@ -161,16 +154,6 @@ migrate-fast: analyze-sqlite: uv run python scripts/migrate_sqlite_to_postgres.py --dry-run -# Distribution -build-exe: - uv run python build_simple.py - -package: build-exe - @echo "Creating distribution package..." - @if exist dist\ping-river-monitor-distribution.zip del dist\ping-river-monitor-distribution.zip - @cd dist && powershell -Command "Compress-Archive -Path * -DestinationPath ping-river-monitor-distribution.zip -Force" - @echo "✅ Distribution package created: dist/ping-river-monitor-distribution.zip" - # Git helpers git-setup: git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git diff --git a/README.md b/README.md index 4ccbf8f..7289d15 100644 --- a/README.md +++ b/README.md @@ -294,20 +294,6 @@ sudo systemctl start water-monitor.service ``` -### Migration for Existing Systems - -If you have an existing installation, use the migration script to add geolocation support: - -```bash -# Stop the service -sudo systemctl stop water-monitor - -# Run migration -python scripts/migrate_geolocation.py - -# Restart the service -sudo systemctl start water-monitor -``` ## 🔧 Command Line Tools @@ -337,18 +323,14 @@ python src/demo_databases.py all # Test all databases ### Core Documentation - **[Data Sources & API Catalog](docs/DATA_SOURCES.md)** - Every ingested and available data source (RID, ThaiWater/HII, dams, rainfall, forecasts) - **[Installation Guide](docs/DATABASE_DEPLOYMENT_GUIDE.md)** - Complete setup instructions -- **[Scheduler Guide](docs/ENHANCED_SCHEDULER_GUIDE.md)** - 15-minute scheduling system -- **[Geolocation Guide](docs/GEOLOCATION_GUIDE.md)** - Grafana geomap integration - **[Gap Filling Guide](docs/GAP_FILLING_GUIDE.md)** - Data integrity management ### Deployment Guides - **[VictoriaMetrics Setup](docs/VICTORIAMETRICS_SETUP.md)** - High-performance deployment -- **[HTTPS Configuration](docs/HTTPS_CONFIGURATION.md)** - Secure deployment - **[Debian Troubleshooting](docs/DEBIAN_TROUBLESHOOTING.md)** - Linux deployment issues ### References - **[Notable Documents](docs/references/NOTABLE_DOCUMENTS.md)** - Official Thai government resources -- **[Migration Guide](docs/MIGRATION_QUICKSTART.md)** - Updating existing systems ## 🔍 Troubleshooting @@ -487,7 +469,7 @@ Northern-Thailand-Ping-River-Monitor/ └── requirements.txt # Dependencies ``` -See [docs/PROJECT_STRUCTURE.md](docs/PROJECT_STRUCTURE.md) for detailed architecture information. +See [docs/FLOOD_FORECASTING.md](docs/FLOOD_FORECASTING.md) for the forecasting architecture and [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md) for the data pipeline. ## 🔄 CI/CD & Automation diff --git a/after b/after new file mode 100644 index 0000000..e69de29 diff --git a/build_executable.py b/build_executable.py deleted file mode 100644 index eac20b0..0000000 --- a/build_executable.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -""" -Build script to create a standalone executable for Northern Thailand Ping River Monitor -""" - -import os -import shutil -import sys -from pathlib import Path - - -def create_spec_file(): - """Create PyInstaller spec file""" - spec_content = """ -# -*- mode: python ; coding: utf-8 -*- - -block_cipher = None - -# Data files to include -data_files = [ - ('.env', '.'), - ('sql/*.sql', 'sql'), - ('README.md', '.'), - ('POSTGRESQL_SETUP.md', '.'), - ('SQLITE_MIGRATION.md', '.'), -] - -# Hidden imports that PyInstaller might miss -hidden_imports = [ - 'psycopg2', - 'psycopg2-binary', - 'sqlalchemy.dialects.postgresql', - 'sqlalchemy.dialects.sqlite', - 'sqlalchemy.dialects.mysql', - 'influxdb', - 'pymysql', - 'dotenv', - 'pydantic', - 'fastapi', - 'uvicorn', - 'schedule', - 'pandas', - 'requests', - 'psutil', -] - -a = Analysis( - ['run.py'], - pathex=['.'], - binaries=[], - datas=data_files, - hiddenimports=hidden_imports, - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[ - 'tkinter', - 'matplotlib', - 'PIL', - 'jupyter', - 'notebook', - 'IPython', - ], - win_no_prefer_redirects=False, - win_private_assemblies=False, - cipher=block_cipher, - noarchive=False, -) - -pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.zipfiles, - a.datas, - [], - name='ping-river-monitor', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, - icon='icon.ico' if os.path.exists('icon.ico') else None, -) -""" - - with open("ping-river-monitor.spec", "w") as f: - f.write(spec_content.strip()) - - print("[OK] Created ping-river-monitor.spec") - - -def install_pyinstaller(): - """Install PyInstaller if not present""" - try: - import PyInstaller - - print("[OK] PyInstaller already installed") - except ImportError: - print("Installing PyInstaller...") - os.system("uv add --dev pyinstaller") - print("[OK] PyInstaller installed") - - -def build_executable(): - """Build the executable""" - print("🔨 Building executable...") - - # Clean previous builds - if os.path.exists("dist"): - shutil.rmtree("dist") - if os.path.exists("build"): - shutil.rmtree("build") - - # Build with PyInstaller using uv - result = os.system("uv run pyinstaller ping-river-monitor.spec --clean --noconfirm") - - if result == 0: - print("✅ Executable built successfully!") - - # Copy additional files to dist directory - dist_dir = Path("dist") - if dist_dir.exists(): - # Copy .env file if it exists - if os.path.exists(".env"): - shutil.copy2(".env", dist_dir / ".env") - print("✅ Copied .env file") - - # Copy documentation - for doc in ["README.md", "POSTGRESQL_SETUP.md", "SQLITE_MIGRATION.md"]: - if os.path.exists(doc): - shutil.copy2(doc, dist_dir / doc) - print(f"✅ Copied {doc}") - - # Copy SQL files - if os.path.exists("sql"): - shutil.copytree("sql", dist_dir / "sql", dirs_exist_ok=True) - print("✅ Copied SQL files") - - print(f"\n🎉 Executable created: {dist_dir / 'ping-river-monitor.exe'}") - print(f"📁 All files in: {dist_dir.absolute()}") - - else: - print("❌ Build failed!") - return False - - return True - - -def create_batch_files(): - """Create convenient batch files""" - batch_files = { - "start.bat": """@echo off -echo Starting Ping River Monitor... -ping-river-monitor.exe -pause -""", - "start-api.bat": """@echo off -echo Starting Ping River Monitor Web API... -ping-river-monitor.exe --web-api -pause -""", - "test.bat": """@echo off -echo Running Ping River Monitor test... -ping-river-monitor.exe --test -pause -""", - "status.bat": """@echo off -echo Checking Ping River Monitor status... -ping-river-monitor.exe --status -pause -""", - } - - dist_dir = Path("dist") - for filename, content in batch_files.items(): - batch_file = dist_dir / filename - with open(batch_file, "w") as f: - f.write(content) - print(f"✅ Created {filename}") - - -def create_readme(): - """Create deployment README""" - readme_content = """# Ping River Monitor - Standalone Executable - -This is a standalone executable version of the Northern Thailand Ping River Monitor. - -## Quick Start - -1. **Configure Database**: Edit `.env` file with your PostgreSQL settings -2. **Test Connection**: Double-click `test.bat` -3. **Start Monitoring**: Double-click `start.bat` -4. **Web Interface**: Double-click `start-api.bat` - -## Files Included - -- `ping-river-monitor.exe` - Main executable -- `.env` - Configuration file (EDIT THIS!) -- `start.bat` - Start continuous monitoring -- `start-api.bat` - Start web API server -- `test.bat` - Run a test cycle -- `status.bat` - Check system status -- `README.md`, `POSTGRESQL_SETUP.md` - Documentation -- `sql/` - Database initialization scripts - -## Configuration - -Edit `.env` file: -``` -DB_TYPE=postgresql -POSTGRES_HOST=your-server-ip -POSTGRES_PORT=5432 -POSTGRES_DB=water_monitoring -POSTGRES_USER=your-username -POSTGRES_PASSWORD=your-password -``` - -## Usage - -### Command Line -```cmd -# Continuous monitoring -ping-river-monitor.exe - -# Single test run -ping-river-monitor.exe --test - -# Web API server -ping-river-monitor.exe --web-api - -# Check status -ping-river-monitor.exe --status -``` - -### Batch Files -- Just double-click the `.bat` files for easy operation - -## Troubleshooting - -1. **Database Connection Issues** - - Check `.env` file settings - - Verify PostgreSQL server is accessible - - Test with `test.bat` - -2. **Permission Issues** - - Run as administrator if needed - - Check firewall settings for API mode - -3. **Log Files** - - Check `water_monitor.log` for detailed logs - - Logs are created in the same directory as the executable - -## Support - -For issues or questions, check the documentation files included. -""" - - with open("dist/DEPLOYMENT_README.txt", "w") as f: - f.write(readme_content) - - print("✅ Created DEPLOYMENT_README.txt") - - -def main(): - """Main build process""" - print("Building Ping River Monitor Executable") - print("=" * 50) - - # Check if we're in the right directory - if not os.path.exists("run.py"): - print( - "❌ Error: run.py not found. Please run this from the project root directory." - ) - return False - - # Install PyInstaller - install_pyinstaller() - - # Create spec file - create_spec_file() - - # Build executable - if not build_executable(): - return False - - # Create convenience files - create_batch_files() - create_readme() - - print("\n" + "=" * 50) - print("🎉 BUILD COMPLETE!") - print("📁 Check the 'dist' folder for your executable") - print("💡 Edit the .env file before distributing") - print("🚀 Ready for deployment!") - - return True - - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) diff --git a/build_simple.py b/build_simple.py deleted file mode 100644 index 631ab86..0000000 --- a/build_simple.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple build script for standalone executable -""" - -import os -import shutil -import sys -from pathlib import Path - - -def main(): - print("Building Ping River Monitor Executable") - print("=" * 50) - - # Check if PyInstaller is installed - try: - import PyInstaller - - print("[OK] PyInstaller available") - except ImportError: - print("[INFO] Installing PyInstaller...") - os.system("uv add --dev pyinstaller") - - # Clean previous builds - if os.path.exists("dist"): - shutil.rmtree("dist") - print("[CLEAN] Removed old dist directory") - if os.path.exists("build"): - shutil.rmtree("build") - print("[CLEAN] Removed old build directory") - - # Build command with all necessary options - cmd = [ - "uv", - "run", - "pyinstaller", - "--onefile", - "--console", - "--name=ping-river-monitor", - "--add-data=.env;.", - "--add-data=sql;sql", - "--add-data=README.md;.", - "--add-data=POSTGRESQL_SETUP.md;.", - "--add-data=SQLITE_MIGRATION.md;.", - "--hidden-import=psycopg2", - "--hidden-import=sqlalchemy.dialects.postgresql", - "--hidden-import=sqlalchemy.dialects.sqlite", - "--hidden-import=dotenv", - "--hidden-import=pydantic", - "--hidden-import=fastapi", - "--hidden-import=uvicorn", - "--hidden-import=schedule", - "--hidden-import=pandas", - "--clean", - "--noconfirm", - "run.py", - ] - - print("[BUILD] Running PyInstaller...") - print("[CMD] " + " ".join(cmd)) - - result = os.system(" ".join(cmd)) - - if result == 0: - print("[SUCCESS] Executable built successfully!") - - # Copy .env file to dist if it exists - if os.path.exists(".env") and os.path.exists("dist"): - shutil.copy2(".env", "dist/.env") - print("[COPY] .env file copied to dist/") - - # Create batch files for easy usage - batch_files = { - "start.bat": """@echo off -echo Starting Ping River Monitor... -ping-river-monitor.exe -pause -""", - "start-api.bat": """@echo off -echo Starting Web API... -ping-river-monitor.exe --web-api -pause -""", - "test.bat": """@echo off -echo Running test... -ping-river-monitor.exe --test -pause -""", - } - - for filename, content in batch_files.items(): - if os.path.exists("dist"): - with open(f"dist/{filename}", "w") as f: - f.write(content) - print(f"[CREATE] {filename}") - - print("\n" + "=" * 50) - print("BUILD COMPLETE!") - print(f"Executable: dist/ping-river-monitor.exe") - print("Batch files: start.bat, start-api.bat, test.bat") - print("Don't forget to edit .env file before using!") - - return True - else: - print("[ERROR] Build failed!") - return False - - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) diff --git a/docs/ENHANCED_SCHEDULER_GUIDE.md b/docs/ENHANCED_SCHEDULER_GUIDE.md deleted file mode 100644 index d0a30b9..0000000 --- a/docs/ENHANCED_SCHEDULER_GUIDE.md +++ /dev/null @@ -1,293 +0,0 @@ -# Enhanced Scheduler Guide - -This guide explains the new 15-minute scheduling system that runs continuously throughout each hour to ensure comprehensive data coverage. - -## ✅ **New Scheduling Behavior** - -### **15-Minute Schedule Pattern** -- **Timing**: Runs every 15 minutes: 1:00, 1:15, 1:30, 1:45, 2:00, 2:15, 2:30, 2:45, etc. -- **Hourly Full Checks**: At :00 minutes (includes gap filling and data updates) -- **Quarter-Hour Quick Checks**: At :15, :30, :45 minutes (data fetch only) -- **Continuous Coverage**: Ensures no data is missed throughout each hour - -### **Operation Types** -- **Full Operations** (at :00): Data fetching + gap filling + data updates -- **Quick Operations** (at :15, :30, :45): Data fetching only for performance - -## 🔧 **Technical Implementation** - -### **Scheduler States** -```python -# State tracking variables -self.last_successful_update = None # Timestamp of last successful data update -self.retry_mode = False # Whether in quick check mode (skip gap filling) -self.next_hourly_check = None # Next scheduled hourly check -``` - -### **Quarter-Hour Check Process** -```python -def quarter_hour_check(self): - """15-minute check for new data""" - current_time = datetime.datetime.now() - minute = current_time.minute - - # Determine if this is a full hourly check (at :00) or a quarter-hour check - if minute == 0: - logging.info("=== HOURLY CHECK (00:00) ===") - self.retry_mode = False # Full check with gap filling and updates - else: - logging.info(f"=== 15-MINUTE CHECK ({minute:02d}:00) ===") - self.retry_mode = True # Skip gap filling and updates on 15-min checks - - new_data_found = self.run_scraping_cycle() - - if new_data_found: - self.last_successful_update = datetime.datetime.now() - if minute == 0: - logging.info("New data found during hourly check") - else: - logging.info(f"New data found during 15-minute check at :{minute:02d}") - else: - if minute == 0: - logging.info("No new data found during hourly check") - else: - logging.info(f"No new data found during 15-minute check at :{minute:02d}") -``` - -### **Scheduler Setup** -```python -def start_scheduler(self): - """Start enhanced scheduler with 15-minute checks""" - # Schedule checks every 15 minutes (at :00, :15, :30, :45) - schedule.every().hour.at(":00").do(self.quarter_hour_check) - schedule.every().hour.at(":15").do(self.quarter_hour_check) - schedule.every().hour.at(":30").do(self.quarter_hour_check) - schedule.every().hour.at(":45").do(self.quarter_hour_check) - - while True: - schedule.run_pending() - time.sleep(30) # Check every 30 seconds -``` - -## 📊 **New Data Detection Logic** - -### **Smart Detection Algorithm** -```python -def has_new_data(self) -> bool: - """Check if there is new data available since last successful update""" - # Get most recent timestamp from database - latest_data = self.get_latest_data(limit=1) - - # Check if we should have newer data by now - now = datetime.datetime.now() - expected_latest = now.replace(minute=0, second=0, microsecond=0) - - # If current time is past 5 minutes after the hour, we should have data - if now.minute >= 5: - if latest_timestamp < expected_latest: - return True # New data expected - - # Check if we have data for the previous hour - previous_hour = expected_latest - datetime.timedelta(hours=1) - if latest_timestamp < previous_hour: - return True # Missing recent data - - return False # Data is up to date -``` - -### **Actual Data Verification** -```python -# Compare timestamps before and after scraping -initial_timestamp = get_latest_timestamp_before_scraping() -# ... perform scraping ... -latest_timestamp = get_latest_timestamp_after_scraping() - -if initial_timestamp is None or latest_timestamp > initial_timestamp: - new_data_found = True - self.last_successful_update = datetime.datetime.now() -``` - -## 🚀 **Operational Modes** - -### **Mode 1: Full Hourly Operation (at :00)** -- **Schedule**: Every hour at :00 minutes (1:00, 2:00, 3:00, etc.) -- **Operations**: - - ✅ Fetch current data - - ✅ Fill data gaps (last 7 days) - - ✅ Update existing data (last 2 days) -- **Purpose**: Comprehensive data collection and maintenance - -### **Mode 2: Quick 15-Minute Checks (at :15, :30, :45)** -- **Schedule**: Every 15 minutes at quarter-hour marks -- **Operations**: - - ✅ Fetch current data only - - ❌ Skip gap filling (performance optimization) - - ❌ Skip data updates (performance optimization) -- **Purpose**: Ensure no new data is missed between hourly checks - -## 📋 **Logging Output Examples** - -### **Successful Hourly Check (at :00)** -``` -2025-07-26 01:00:00,123 - INFO - === HOURLY CHECK (00:00) === -2025-07-26 01:00:00,124 - INFO - Starting scraping cycle... -2025-07-26 01:00:01,456 - INFO - Successfully fetched 384 data points from API -2025-07-26 01:00:02,789 - INFO - New data found: 2025-07-26 01:00:00 -2025-07-26 01:00:03,012 - INFO - Filled 5 data gaps -2025-07-26 01:00:04,234 - INFO - Updated 2 existing measurements -2025-07-26 01:00:04,235 - INFO - New data found during hourly check -``` - -### **15-Minute Quick Check (at :15, :30, :45)** -``` -2025-07-26 01:15:00,123 - INFO - === 15-MINUTE CHECK (15:00) === -2025-07-26 01:15:00,124 - INFO - Starting scraping cycle... -2025-07-26 01:15:01,456 - INFO - Successfully fetched 299 data points from API -2025-07-26 01:15:02,789 - INFO - New data found: 2025-07-26 01:00:00 -2025-07-26 01:15:02,790 - INFO - New data found during 15-minute check at :15 -``` - -### **Continuous 15-Minute Pattern** -``` -2025-07-26 01:00:00,123 - INFO - === HOURLY CHECK (00:00) === -2025-07-26 01:00:04,235 - INFO - New data found during hourly check - -2025-07-26 01:15:00,123 - INFO - === 15-MINUTE CHECK (15:00) === -2025-07-26 01:15:02,790 - INFO - No new data found during 15-minute check at :15 - -2025-07-26 01:30:00,123 - INFO - === 15-MINUTE CHECK (30:00) === -2025-07-26 01:30:02,790 - INFO - No new data found during 15-minute check at :30 - -2025-07-26 01:45:00,123 - INFO - === 15-MINUTE CHECK (45:00) === -2025-07-26 01:45:02,790 - INFO - No new data found during 15-minute check at :45 - -2025-07-26 02:00:00,123 - INFO - === HOURLY CHECK (00:00) === -2025-07-26 02:00:04,235 - INFO - New data found during hourly check -``` - -## ⚙️ **Configuration Options** - -### **Environment Variables** -```bash -# Retry interval (default: 5 minutes) -export RETRY_INTERVAL_MINUTES=5 - -# Data availability buffer (default: 5 minutes after hour) -export DATA_BUFFER_MINUTES=5 - -# Gap filling days (default: 7 days) -export GAP_FILL_DAYS=7 - -# Update check days (default: 2 days) -export UPDATE_DAYS=2 -``` - -### **Scheduler Timing** -```python -# Hourly checks at top of hour -schedule.every().hour.at(":00").do(self.hourly_check) - -# 5-minute retries (dynamically scheduled) -schedule.every(5).minutes.do(self.retry_check).tag('retry') - -# Check every 30 seconds for responsive retry scheduling -time.sleep(30) -``` - -## 🔍 **Performance Optimizations** - -### **Retry Mode Optimizations** -- **Skip Gap Filling**: Avoids expensive historical data fetching during retries -- **Skip Data Updates**: Avoids comparison operations during retries -- **Focused API Calls**: Only fetches current day data during retries -- **Reduced Database Queries**: Minimal database operations during retries - -### **Resource Management** -- **API Rate Limiting**: 1-second delays between API calls -- **Database Connection Pooling**: Efficient connection reuse -- **Memory Efficiency**: Selective data processing -- **Error Recovery**: Automatic retry with exponential backoff - -## 🛠️ **Troubleshooting** - -### **Common Scenarios** - -#### **Stuck in Retry Mode** -``` -# Check if API is returning data -curl -X POST https://hyd-app-db.rid.go.th/webservice/getGroupHourlyWaterLevelReportAllHL.ashx - -# Check database connectivity -python water_scraper_v3.py --check-gaps 1 - -# Manual data fetch test -python water_scraper_v3.py --test -``` - -#### **Missing Hourly Triggers** -``` -# Check system time synchronization -timedatectl status - -# Verify scheduler is running -ps aux | grep water_scraper - -# Check logs for scheduler activity -tail -f water_monitor.log | grep "HOURLY CHECK" -``` - -#### **False New Data Detection** -``` -# Check latest data in database -sqlite3 water_monitoring.db "SELECT MAX(timestamp) FROM water_measurements;" - -# Verify timestamp parsing -python -c " -import datetime -print('Current hour:', datetime.datetime.now().replace(minute=0, second=0, microsecond=0)) -" -``` - -## 📈 **Monitoring and Alerts** - -### **Key Metrics to Monitor** -- **Hourly Success Rate**: Percentage of hourly checks that find new data -- **Retry Duration**: How long system stays in retry mode -- **Data Freshness**: Time since last successful data update -- **API Response Time**: Performance of data fetching operations - -### **Alert Conditions** -- **Extended Retry Mode**: System in retry mode for > 30 minutes -- **No Data for 2+ Hours**: No new data found for extended period -- **High Error Rate**: Multiple consecutive API failures -- **Database Issues**: Connection or save failures - -### **Health Check Script** -```bash -#!/bin/bash -# Check if system is stuck in retry mode -RETRY_COUNT=$(tail -n 100 water_monitor.log | grep -c "RETRY CHECK") -if [ $RETRY_COUNT -gt 6 ]; then - echo "WARNING: System may be stuck in retry mode ($RETRY_COUNT retries in last 100 log entries)" -fi - -# Check data freshness -LATEST_DATA=$(sqlite3 water_monitoring.db "SELECT MAX(timestamp) FROM water_measurements;") -echo "Latest data timestamp: $LATEST_DATA" -``` - -## 🎯 **Best Practices** - -### **Production Deployment** -1. **Monitor Logs**: Watch for retry mode patterns -2. **Set Alerts**: Configure notifications for extended retry periods -3. **Regular Maintenance**: Weekly gap filling and data validation -4. **Backup Strategy**: Regular database backups before major operations - -### **Performance Tuning** -1. **Adjust Buffer Time**: Modify data availability buffer based on API patterns -2. **Optimize Retry Interval**: Balance between responsiveness and API load -3. **Database Indexing**: Ensure proper indexes for timestamp queries -4. **Connection Pooling**: Configure appropriate database connection limits - -This enhanced scheduler ensures reliable, efficient, and intelligent water level monitoring with automatic adaptation to data availability patterns. diff --git a/docs/ENHANCEMENT_SUMMARY.md b/docs/ENHANCEMENT_SUMMARY.md deleted file mode 100644 index ab123a8..0000000 --- a/docs/ENHANCEMENT_SUMMARY.md +++ /dev/null @@ -1,227 +0,0 @@ -# 🚀 Northern Thailand Ping River Monitor - Enhancement Summary - -## 🎯 **What We've Accomplished** - -We've successfully transformed your water monitoring system from a simple scraper into a **production-ready, enterprise-grade monitoring platform** focused on the Ping River Basin in Northern Thailand, with modern web interfaces, station management capabilities, and comprehensive observability. - -## 🌟 **Major New Features Added** - -### 1. **FastAPI Web Interface** 🌐 -- **Interactive Dashboard** at `http://localhost:8000` -- **REST API** with comprehensive endpoints -- **Station Management** - Add, update, delete monitoring stations -- **Real-time Health Monitoring** -- **Manual Data Collection Triggers** -- **Interactive API Documentation** at `/docs` -- **CORS Support** for web applications - -### 2. **Enhanced Architecture** 🏗️ -- **Type Safety** with Pydantic models and comprehensive type hints -- **Data Validation Layer** with range checking and error handling -- **Custom Exception Classes** for better error management -- **Modular Design** with separated concerns - -### 3. **Observability & Monitoring** 📊 -- **Metrics Collection System** (counters, gauges, histograms) -- **Health Checks** for database, API, and system resources -- **Performance Tracking** with response times and success rates -- **Enhanced Logging** with colors, rotation, and performance logs - -### 4. **Production Features** 🚀 -- **Rate Limiting** to prevent API abuse -- **Request Tracking** with detailed statistics -- **Configuration Validation** on startup -- **Graceful Error Handling** and recovery -- **Background Task Management** - -## 📁 **New Files Created** - -``` -src/ -├── models.py # Data models and type definitions -├── exceptions.py # Custom exception classes -├── validators.py # Data validation layer -├── metrics.py # Metrics collection system -├── health_check.py # Health monitoring system -├── rate_limiter.py # Rate limiting and request tracking -├── logging_config.py # Enhanced logging configuration -├── web_api.py # FastAPI web interface -├── main.py # Enhanced CLI with multiple modes -└── __init__.py # Package initialization - -# Root files -├── run.py # Simple startup script -├── test_integration.py # Integration test suite -├── test_api.py # API endpoint tests -└── ENHANCEMENT_SUMMARY.md # This file -``` - -## 🔧 **Enhanced Existing Files** - -- **`src/water_scraper_v3.py`** - Integrated new features, metrics, validation -- **`src/config.py`** - Added configuration validation -- **`requirements.txt`** - Added FastAPI, Pydantic, and monitoring dependencies -- **`docker-compose.victoriametrics.yml`** - Added web API service -- **`Dockerfile`** - Updated for new startup script -- **`README.md`** - Updated with new features and usage instructions - -## 🌐 **Web API Endpoints** - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/` | GET | Interactive dashboard | -| `/docs` | GET | API documentation | -| `/health` | GET | System health status | -| `/metrics` | GET | Application metrics | -| `/stations` | GET | List all monitoring stations | -| `/measurements/latest` | GET | Latest measurements | -| `/measurements/station/{code}` | GET | Station-specific data | -| `/scrape/trigger` | POST | Trigger manual data collection | -| `/scraping/status` | GET | Scraping status and statistics | -| `/config` | GET | Current configuration (masked) | - -## 🚀 **Usage Examples** - -### **Traditional Mode (Enhanced)** -```bash -# Test single cycle -python run.py --test - -# Continuous monitoring -python run.py - -# Fill data gaps -python run.py --fill-gaps 7 - -# Show system status -python run.py --status -``` - -### **Web API Mode (NEW!)** -```bash -# Start web API server -python run.py --web-api - -# Access dashboard -open http://localhost:8000 - -# View API documentation -open http://localhost:8000/docs -``` - -### **Docker Deployment** -```bash -# Start complete stack -docker-compose -f docker-compose.victoriametrics.yml up -d - -# Services available: -# - Water API: http://localhost:8000 -# - Grafana: http://localhost:3000 -# - VictoriaMetrics: http://localhost:8428 -``` - -## 📊 **Monitoring & Observability** - -### **Built-in Metrics** -- API request counts and response times -- Database connection status and save operations -- Scraping cycle success/failure rates -- System resource usage (memory, etc.) - -### **Health Checks** -- Database connectivity and data freshness -- External API availability -- Memory usage monitoring -- Overall system health status - -### **Enhanced Logging** -- Colored console output for better readability -- File rotation to prevent disk space issues -- Performance logging for optimization -- Structured logging with proper levels - -## 🔒 **Production Ready Features** - -### **Security & Reliability** -- Rate limiting to prevent API abuse -- Input validation and sanitization -- Graceful error handling and recovery -- Configuration validation on startup - -### **Performance** -- Efficient metrics collection with minimal overhead -- Background task management -- Connection pooling and resource management -- Optimized database operations - -### **Scalability** -- Modular architecture for easy extension -- Async support for high concurrency -- Configurable resource limits -- Health checks for load balancer integration - -## 🧪 **Testing** - -### **Integration Tests** -```bash -# Run all integration tests -python test_integration.py -``` - -### **API Tests** -```bash -# Test API endpoints (server must be running) -python test_api.py -``` - -## 📈 **Performance Improvements** - -1. **Request Tracking** - Monitor API performance and success rates -2. **Rate Limiting** - Prevent API abuse and ensure stability -3. **Data Validation** - Catch errors early and improve data quality -4. **Metrics Collection** - Identify bottlenecks and optimization opportunities -5. **Health Monitoring** - Proactive issue detection and alerting - -## 🎉 **Benefits Achieved** - -### **For Developers** -- **Better Developer Experience** with type hints and validation -- **Easier Debugging** with enhanced logging and error messages -- **Comprehensive Testing** with integration and API tests -- **Modern Architecture** following best practices - -### **For Operations** -- **Web Dashboard** for easy monitoring and management -- **Health Checks** for automated monitoring integration -- **Metrics Collection** for performance analysis -- **Production-Ready** deployment with Docker support - -### **For Users** -- **REST API** for integration with other systems -- **Real-time Data Access** via web interface -- **Manual Controls** for triggering data collection -- **Status Monitoring** for system visibility - -## 🔮 **Future Enhancement Opportunities** - -1. **Authentication & Authorization** - Add user management and API keys -2. **Real-time WebSocket Updates** - Live data streaming to web clients -3. **Advanced Analytics** - Trend analysis and forecasting -4. **Alert System** - Email/SMS notifications for critical conditions -5. **Multi-tenant Support** - Support for multiple organizations -6. **Data Export** - CSV, Excel, and other format exports -7. **Mobile App** - React Native or Flutter mobile interface - -## 🏆 **Summary** - -Your Thailand Water Monitor has been transformed from a simple data scraper into a **comprehensive, enterprise-grade monitoring platform** that includes: - -- ✅ **Modern Web Interface** with FastAPI -- ✅ **Production-Ready Architecture** with proper error handling -- ✅ **Comprehensive Monitoring** with metrics and health checks -- ✅ **Type Safety** and data validation -- ✅ **Enhanced Logging** and observability -- ✅ **Docker Support** for easy deployment -- ✅ **Extensive Testing** for reliability - -The system is now ready for production deployment and can serve as a foundation for further enhancements and integrations! \ No newline at end of file diff --git a/docs/GEOLOCATION_GUIDE.md b/docs/GEOLOCATION_GUIDE.md deleted file mode 100644 index 9c06eef..0000000 --- a/docs/GEOLOCATION_GUIDE.md +++ /dev/null @@ -1,475 +0,0 @@ -# Geolocation Support for Grafana Geomap - -This guide explains the geolocation functionality added to the Thailand Water Monitor for use with Grafana's geomap visualization. - -## ✅ **Implemented Features** - -### **Database Schema Updates** -All database adapters now support geolocation fields: -- **latitude**: Decimal latitude coordinates (DECIMAL(10,8) for SQL, REAL for SQLite) -- **longitude**: Decimal longitude coordinates (DECIMAL(11,8) for SQL, REAL for SQLite) -- **geohash**: Geohash string for efficient spatial indexing (VARCHAR(20)/TEXT) - -### **Station Data Enhancement** -Station mapping now includes geolocation fields: -```python -'8': { - 'code': 'P.1', - 'thai_name': 'สะพานนวรัฐ', - 'english_name': 'Nawarat Bridge', - 'latitude': 15.6944, # Decimal degrees - 'longitude': 100.2028, # Decimal degrees - 'geohash': 'w5q6uuhvfcfp25' # Geohash for P.1 -} -``` - -## 🗄️ **Database Schema** - -### **Updated Stations Table** -```sql -CREATE TABLE stations ( - id INTEGER PRIMARY KEY, - station_code TEXT UNIQUE NOT NULL, - thai_name TEXT NOT NULL, - english_name TEXT NOT NULL, - latitude REAL, -- NEW: Latitude coordinate - longitude REAL, -- NEW: Longitude coordinate - geohash TEXT, -- NEW: Geohash for spatial indexing - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); -``` - -### **Database Support** -- ✅ **SQLite**: REAL columns for coordinates, TEXT for geohash -- ✅ **PostgreSQL**: DECIMAL(10,8) and DECIMAL(11,8) for coordinates, VARCHAR(20) for geohash -- ✅ **MySQL**: DECIMAL(10,8) and DECIMAL(11,8) for coordinates, VARCHAR(20) for geohash -- ✅ **VictoriaMetrics**: Geolocation data included in metric labels - -## 📊 **Current Station Data** - -### **P.1 - Nawarat Bridge (Sample)** -- **Station Code**: P.1 -- **Thai Name**: สะพานนวรัฐ -- **English Name**: Nawarat Bridge -- **Latitude**: 15.6944 -- **Longitude**: 100.2028 -- **Geohash**: w5q6uuhvfcfp25 - -### **Remaining Stations** -The following stations are ready for geolocation data when coordinates become available: -- P.20 - บ้านเชียงดาว (Ban Chiang Dao) -- P.75 - บ้านช่อแล (Ban Chai Lat) -- P.92 - บ้านเมืองกึ๊ด (Ban Muang Aut) -- P.4A - บ้านแม่แตง (Ban Mae Taeng) -- P.67 - บ้านแม่แต (Ban Tae) -- P.21 - บ้านริมใต้ (Ban Rim Tai) -- P.103 - สะพานวงแหวนรอบ 3 (Ring Bridge 3) -- P.82 - บ้านสบวิน (Ban Sob win) -- P.84 - บ้านพันตน (Ban Panton) -- P.81 - บ้านโป่ง (Ban Pong) -- P.5 - สะพานท่านาง (Tha Nang Bridge) -- P.77 - บ้านสบแม่สะป๊วด (Baan Sop Mae Sapuord) -- P.87 - บ้านป่าซาง (Ban Pa Sang) -- P.76 - บ้านแม่อีไฮ (Banb Mae I Hai) -- P.85 - บ้านหล่ายแก้ว (Baan Lai Kaew) - -## 🗺️ **Grafana Geomap Integration** - -### **Data Source Configuration** -The geolocation data is automatically included in all database queries and can be used directly in Grafana: - -#### **SQLite/PostgreSQL/MySQL Query Example** -```sql -SELECT - m.timestamp, - s.station_code, - s.english_name, - s.thai_name, - s.latitude, - s.longitude, - s.geohash, - m.water_level, - m.discharge, - m.discharge_percent -FROM water_measurements m -JOIN stations s ON m.station_id = s.id -WHERE s.latitude IS NOT NULL - AND s.longitude IS NOT NULL -ORDER BY m.timestamp DESC -``` - -#### **VictoriaMetrics Query Example** -```promql -water_level{latitude!="",longitude!=""} -``` - -### **Geomap Panel Configuration** - -#### **1. Create Geomap Panel** -1. Add new panel in Grafana -2. Select "Geomap" visualization -3. Configure data source (SQLite/PostgreSQL/MySQL/VictoriaMetrics) - -#### **2. Configure Location Fields** -- **Latitude Field**: `latitude` -- **Longitude Field**: `longitude` -- **Alternative**: Use `geohash` field for geohash-based positioning - -#### **3. Configure Display Options** -- **Station Labels**: Use `station_code` or `english_name` -- **Tooltip Information**: Include `thai_name`, `water_level`, `discharge` -- **Color Mapping**: Map to `water_level` or `discharge_percent` - -#### **4. Sample Geomap Configuration** -```json -{ - "type": "geomap", - "title": "Thailand Water Stations", - "targets": [ - { - "rawSql": "SELECT latitude, longitude, station_code, english_name, water_level, discharge_percent FROM stations s JOIN water_measurements m ON s.id = m.station_id WHERE s.latitude IS NOT NULL AND m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id)", - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "color": { - "mode": "continuous-GrYlRd", - "field": "water_level" - } - } - }, - "options": { - "view": { - "id": "coords", - "lat": 15.6944, - "lon": 100.2028, - "zoom": 8 - }, - "controls": { - "mouseWheelZoom": true, - "showZoom": true, - "showAttribution": true - }, - "layers": [ - { - "type": "markers", - "config": { - "size": { - "field": "discharge_percent", - "min": 5, - "max": 20 - }, - "color": { - "field": "water_level" - }, - "showLegend": true - } - } - ] - } -} -``` - -## 🔧 **Adding New Station Coordinates** - -### **Method 1: Update Station Mapping** -Edit `water_scraper_v3.py` and add coordinates to the station mapping: -```python -'1': { - 'code': 'P.20', - 'thai_name': 'บ้านเชียงดาว', - 'english_name': 'Ban Chiang Dao', - 'latitude': 19.3056, # Add actual coordinates - 'longitude': 98.9264, # Add actual coordinates - 'geohash': 'w4r6...' # Add actual geohash -} -``` - -### **Method 2: Direct Database Update** -```sql -UPDATE stations -SET latitude = 19.3056, longitude = 98.9264, geohash = 'w4r6uuhvfcfp25' -WHERE station_code = 'P.20'; -``` - -### **Method 3: Bulk Update Script** -```python -import sqlite3 - -coordinates = { - 'P.20': {'lat': 19.3056, 'lon': 98.9264, 'geohash': 'w4r6uuhvfcfp25'}, - 'P.75': {'lat': 18.7756, 'lon': 99.1234, 'geohash': 'w4r5uuhvfcfp25'}, - # Add more stations... -} - -conn = sqlite3.connect('water_monitoring.db') -cursor = conn.cursor() - -for station_code, coords in coordinates.items(): - cursor.execute(""" - UPDATE stations - SET latitude = ?, longitude = ?, geohash = ? - WHERE station_code = ? - """, (coords['lat'], coords['lon'], coords['geohash'], station_code)) - -conn.commit() -conn.close() -``` - -## 🌐 **Geohash Information** - -### **What is Geohash?** -Geohash is a geocoding system that represents geographic coordinates as a short alphanumeric string. It provides: -- **Spatial Indexing**: Efficient spatial queries -- **Proximity**: Similar geohashes indicate nearby locations -- **Hierarchical**: Longer geohashes provide more precision - -### **Geohash Precision Levels** -- **5 characters**: ~2.4km precision -- **6 characters**: ~610m precision -- **7 characters**: ~76m precision -- **8 characters**: ~19m precision -- **9+ characters**: <5m precision - -### **Example: P.1 Geohash** -- **Geohash**: `w5q6uuhvfcfp25` -- **Length**: 14 characters -- **Precision**: Sub-meter accuracy -- **Location**: Nawarat Bridge, Thailand - -## 📈 **Grafana Visualization Examples** - -### **1. Station Location Map** -- **Type**: Geomap with markers -- **Data**: Current station locations -- **Color**: Water level or discharge percentage -- **Size**: Discharge volume - -### **2. Regional Water Levels** -- **Type**: Geomap with heatmap -- **Data**: Water level data across regions -- **Visualization**: Color-coded intensity map -- **Filters**: Time range, station groups - -### **3. Alert Zones** -- **Type**: Geomap with threshold markers -- **Data**: Stations exceeding alert thresholds -- **Visualization**: Red markers for high water levels -- **Alerts**: Automated notifications for critical levels - -## 🔄 **Updating a Running System** - -### **Automated Migration Script** -Use the provided migration script to safely add geolocation columns to your existing database: - -```bash -# Stop the water monitoring service first -sudo systemctl stop water-monitor - -# Run the migration script -python migrate_geolocation.py - -# Restart the service -sudo systemctl start water-monitor -``` - -### **Migration Script Features** -- ✅ **Auto-detects database type** from environment variables -- ✅ **Checks existing columns** to avoid conflicts -- ✅ **Supports all database types** (SQLite, PostgreSQL, MySQL) -- ✅ **Adds sample data** for P.1 station -- ✅ **Safe operation** - won't break existing data - -### **Step-by-Step Migration Process** - -#### **1. Stop the Application** -```bash -# If running as systemd service -sudo systemctl stop water-monitor - -# If running in screen/tmux -# Use Ctrl+C to stop the process - -# If running as Docker container -docker stop water-monitor -``` - -#### **2. Backup Your Database** -```bash -# SQLite backup -cp water_monitoring.db water_monitoring.db.backup - -# PostgreSQL backup -pg_dump water_monitoring > water_monitoring_backup.sql - -# MySQL backup -mysqldump water_monitoring > water_monitoring_backup.sql -``` - -#### **3. Run Migration Script** -```bash -# Default (uses environment variables) -python migrate_geolocation.py - -# Or specify database path for SQLite -SQLITE_DB_PATH=/path/to/water_monitoring.db python migrate_geolocation.py -``` - -#### **4. Verify Migration** -```bash -# Check SQLite schema -sqlite3 water_monitoring.db ".schema stations" - -# Check PostgreSQL schema -psql -d water_monitoring -c "\d stations" - -# Check MySQL schema -mysql -e "DESCRIBE water_monitoring.stations" -``` - -#### **5. Update Application Code** -Ensure you have the latest version of the application with geolocation support: -```bash -# Pull latest code -git pull origin main - -# Install any new dependencies -pip install -r requirements.txt -``` - -#### **6. Restart Application** -```bash -# Systemd service -sudo systemctl start water-monitor - -# Docker container -docker start water-monitor - -# Manual execution -python water_scraper_v3.py -``` - -### **Migration Output Example** -``` -2025-07-28 17:30:00,123 - INFO - Starting geolocation column migration... -2025-07-28 17:30:00,124 - INFO - Detected database type: SQLITE -2025-07-28 17:30:00,125 - INFO - Migrating SQLite database: water_monitoring.db -2025-07-28 17:30:00,126 - INFO - Current columns in stations table: ['id', 'station_code', 'thai_name', 'english_name', 'created_at', 'updated_at'] -2025-07-28 17:30:00,127 - INFO - Added latitude column -2025-07-28 17:30:00,128 - INFO - Added longitude column -2025-07-28 17:30:00,129 - INFO - Added geohash column -2025-07-28 17:30:00,130 - INFO - Successfully added columns: latitude, longitude, geohash -2025-07-28 17:30:00,131 - INFO - Updated P.1 station with sample geolocation data -2025-07-28 17:30:00,132 - INFO - P.1 station geolocation: ('P.1', 15.6944, 100.2028, 'w5q6uuhvfcfp25') -2025-07-28 17:30:00,133 - INFO - ✅ Migration completed successfully! -2025-07-28 17:30:00,134 - INFO - You can now restart your water monitoring application -2025-07-28 17:30:00,135 - INFO - The system will automatically use the new geolocation columns -``` - -## 🔍 **Troubleshooting** - -### **Migration Issues** - -#### **Database Locked Error** -```bash -# Stop all processes using the database -sudo systemctl stop water-monitor -pkill -f water_scraper - -# Wait a few seconds, then run migration -sleep 5 -python migrate_geolocation.py -``` - -#### **Permission Denied** -```bash -# Check database file permissions -ls -la water_monitoring.db - -# Fix permissions if needed -sudo chown $USER:$USER water_monitoring.db -chmod 664 water_monitoring.db -``` - -#### **Missing Dependencies** -```bash -# For PostgreSQL -pip install psycopg2-binary - -# For MySQL -pip install pymysql - -# For all databases -pip install -r requirements.txt -``` - -### **Verification Issues** - -#### **Missing Coordinates** -If stations don't appear on the geomap: -1. Check if latitude/longitude are NULL in database -2. Verify geolocation data in station mapping -3. Ensure database schema includes geolocation columns -4. Run migration script if columns are missing - -#### **Incorrect Positioning** -If stations appear in wrong locations: -1. Verify coordinate format (decimal degrees) -2. Check latitude/longitude order (lat first, lon second) -3. Validate geohash accuracy - -### **Rollback Procedure** -If migration causes issues: - -#### **SQLite Rollback** -```bash -# Stop application -sudo systemctl stop water-monitor - -# Restore backup -cp water_monitoring.db.backup water_monitoring.db - -# Restart with old version -sudo systemctl start water-monitor -``` - -#### **PostgreSQL Rollback** -```sql --- Remove added columns -ALTER TABLE stations DROP COLUMN IF EXISTS latitude; -ALTER TABLE stations DROP COLUMN IF EXISTS longitude; -ALTER TABLE stations DROP COLUMN IF EXISTS geohash; -``` - -#### **MySQL Rollback** -```sql --- Remove added columns -ALTER TABLE stations DROP COLUMN latitude; -ALTER TABLE stations DROP COLUMN longitude; -ALTER TABLE stations DROP COLUMN geohash; -``` - -## 🎯 **Next Steps** - -### **Immediate Actions** -1. **Gather Coordinates**: Collect GPS coordinates for all 16 stations -2. **Update Database**: Add coordinates to remaining stations -3. **Create Dashboards**: Build Grafana geomap visualizations - -### **Future Enhancements** -1. **Automatic Geocoding**: API integration for address-to-coordinate conversion -2. **Mobile GPS**: Mobile app for field coordinate collection -3. **Satellite Integration**: Satellite imagery overlay in Grafana -4. **Geofencing**: Alert zones based on geographic boundaries - -The geolocation functionality is now fully implemented and ready for use with Grafana's geomap visualization. Station P.1 (Nawarat Bridge) serves as a working example with complete coordinate data. diff --git a/docs/GITEA_WORKFLOWS.md b/docs/GITEA_WORKFLOWS.md index 4254e83..f4a327e 100644 --- a/docs/GITEA_WORKFLOWS.md +++ b/docs/GITEA_WORKFLOWS.md @@ -286,8 +286,6 @@ make validate-workflows ### **Project-Specific Resources** - [Contributing Guide](../CONTRIBUTING.md) -- [Deployment Checklist](../DEPLOYMENT_CHECKLIST.md) -- [Project Structure](PROJECT_STRUCTURE.md) ### **Monitoring and Alerts** - Workflow status badges in README diff --git a/docs/GRAFANA_MATRIX_ALERTING.md b/docs/GRAFANA_MATRIX_ALERTING.md deleted file mode 100644 index ef2e98f..0000000 --- a/docs/GRAFANA_MATRIX_ALERTING.md +++ /dev/null @@ -1,168 +0,0 @@ -# Grafana Matrix Alerting Setup - -## Overview -Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded. - -## Prerequisites -- Grafana instance with your PostgreSQL data source -- Matrix account and access token -- Matrix room for alerts - -## Step 1: Configure Matrix Contact Point - -1. **In Grafana, go to Alerting → Contact Points** -2. **Add new contact point:** - ``` - Name: matrix-water-alerts - Integration: Webhook - URL: https://matrix.org/_matrix/client/v3/rooms/!ROOM_ID:matrix.org/send/m.room.message - HTTP Method: POST - ``` - -3. **Add Headers:** - ``` - Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN - Content-Type: application/json - ``` - -4. **Message Template:** - ```json - { - "msgtype": "m.text", - "body": "🌊 WATER ALERT: {{ .CommonLabels.alertname }}\n\nStation: {{ .CommonLabels.station_code }}\nLevel: {{ .CommonAnnotations.water_level }}m\nStatus: {{ .CommonLabels.severity }}\n\nTime: {{ .CommonAnnotations.time }}" - } - ``` - -## Step 2: Create Alert Rules - -### High Water Level Alert -```yaml -Rule Name: high-water-level -Query: water_level > 6.0 -Condition: IS ABOVE 6.0 FOR 5m -Labels: - - severity: critical - - station_code: {{ .station_code }} -Annotations: - - water_level: {{ .water_level }} - - summary: "Critical water level at {{ .station_code }}" -``` - -### Low Water Level Alert -```yaml -Rule Name: low-water-level -Query: water_level < 1.0 -Condition: IS BELOW 1.0 FOR 10m -Labels: - - severity: warning - - station_code: {{ .station_code }} -``` - -### Data Gap Alert -```yaml -Rule Name: data-gap -Query: increase(measurements_total[1h]) == 0 -Condition: IS EQUAL TO 0 FOR 30m -Labels: - - severity: warning - - issue: data-gap -``` - -## Step 3: Matrix Setup - -### Get Matrix Access Token -```bash -curl -X POST https://matrix.org/_matrix/client/v3/login \ - -H "Content-Type: application/json" \ - -d '{ - "type": "m.login.password", - "user": "your_username", - "password": "your_password" - }' -``` - -### Create Alert Room -```bash -curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \ - -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "name": "Water Level Alerts - Northern Thailand", - "topic": "Automated alerts for Ping River water monitoring", - "preset": "trusted_private_chat" - }' -``` - -## Example Alert Queries - -### Critical Water Levels -```promql -# High water alert -water_level{station_code=~"P.1|P.4A|P.20"} > 6.0 - -# Dangerous discharge -discharge{station_code=~".*"} > 500 - -# Rapid level change -increase(water_level[15m]) > 0.5 -``` - -### System Health -```promql -# No data received -up{job="water-monitor"} == 0 - -# Old data -(time() - timestamp) > 7200 -``` - -## Alert Notification Format - -Your Matrix messages will look like: -``` -🌊 WATER ALERT: High Water Level - -Station: P.1 (Chiang Mai) -Level: 6.2m (CRITICAL) -Discharge: 450 cms -Status: DANGER - -Time: 2025-09-26 14:30:00 -Trend: Rising (+0.3m in 30min) - -📍 Location: 18.7883°N, 98.9853°E -``` - -## Advanced Features - -### Escalation Rules -```yaml -# Send to different rooms based on severity -- if: severity == "critical" - receiver: matrix-emergency -- if: severity == "warning" - receiver: matrix-alerts -- if: time_of_day() outside "08:00-20:00" - receiver: matrix-night-duty -``` - -### Rate Limiting -```yaml -group_wait: 5m -group_interval: 10m -repeat_interval: 30m -``` - -## Testing Alerts - -1. **Test Contact Point** - Use Grafana's test button -2. **Simulate Alert** - Manually trigger with test data -3. **Verify Matrix** - Check message formatting and delivery - -## Troubleshooting - -### Common Issues -- **403 Forbidden**: Check Matrix access token -- **Room not found**: Verify room ID format -- **No alerts**: Check query syntax and thresholds -- **Spam**: Configure proper grouping and intervals \ No newline at end of file diff --git a/docs/GRAFANA_MATRIX_SETUP.md b/docs/GRAFANA_MATRIX_SETUP.md deleted file mode 100644 index 6db7201..0000000 --- a/docs/GRAFANA_MATRIX_SETUP.md +++ /dev/null @@ -1,351 +0,0 @@ -# Complete Grafana Matrix Alerting Setup Guide - -## Overview -Configure Grafana to send water level alerts directly to Matrix channels when thresholds are exceeded. - -## Prerequisites -- Grafana instance running (v8.0+) -- PostgreSQL data source configured in Grafana -- Matrix account -- Matrix room for alerts - -## Step 1: Get Matrix Access Token - -### Method 1: Using curl -```bash -curl -X POST https://matrix.org/_matrix/client/v3/login \ - -H "Content-Type: application/json" \ - -d '{ - "type": "m.login.password", - "user": "your_username", - "password": "your_password" - }' -``` - -### Method 2: Using Element Web Client -1. Open Element in browser: https://app.element.io -2. Login to your account -3. Go to Settings → Help & About → Advanced -4. Copy your Access Token - -### Method 3: Using Matrix Admin Panel -- If you have admin access to your homeserver, generate token via admin API - -## Step 2: Create Alert Room - -```bash -curl -X POST "https://matrix.org/_matrix/client/v3/createRoom" \ - -H "Authorization: Bearer YOUR_ACCESS_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "name": "Water Level Alerts - Northern Thailand", - "topic": "Automated alerts for Ping River water monitoring", - "preset": "private_chat" - }' -``` - -Save the `room_id` from the response (format: !roomid:homeserver.com) - -## Step 3: Configure Grafana Contact Point - -### Navigate to Alerting -1. In Grafana, go to **Alerting → Contact Points** -2. Click **Add contact point** - -### Contact Point Settings -``` -Name: matrix-water-alerts -Integration: Webhook -URL: https://matrix.org/_matrix/client/v3/rooms/!YOUR_ROOM_ID:matrix.org/send/m.room.message/{{ .GroupLabels.alertname }}_{{ .GroupLabels.severity }}_{{ now.Unix }} -HTTP Method: POST -``` - -### Headers -``` -Authorization: Bearer YOUR_MATRIX_ACCESS_TOKEN -Content-Type: application/json -``` - -### Message Template (JSON Body) -```json -{ - "msgtype": "m.text", - "body": "🌊 **PING RIVER WATER ALERT**\n\n**Alert:** {{ .GroupLabels.alertname }}\n**Severity:** {{ .GroupLabels.severity | toUpper }}\n**Station:** {{ .GroupLabels.station_code }} ({{ .GroupLabels.station_name }})\n\n{{ range .Alerts }}**Status:** {{ .Status | toUpper }}\n**Water Level:** {{ .Annotations.water_level }}m\n**Threshold:** {{ .Annotations.threshold }}m\n**Time:** {{ .StartsAt.Format \"2006-01-02 15:04:05\" }}\n{{ if .Annotations.discharge }}**Discharge:** {{ .Annotations.discharge }} cms\n{{ end }}{{ if .Annotations.message }}**Details:** {{ .Annotations.message }}\n{{ end }}{{ end }}\n📈 **Dashboard:** {{ .ExternalURL }}\n📍 **Location:** Northern Thailand Ping River" -} -``` - -## Step 4: Create Alert Rules - -### High Water Level Alert -```yaml -# Rule Configuration -Rule Name: high-water-level -Evaluation Group: water-level-alerts -Folder: Water Monitoring - -# Query A -SELECT - station_code, - station_name_th as station_name, - water_level, - discharge, - timestamp -FROM water_measurements -WHERE - timestamp > now() - interval '5 minutes' - AND water_level > 6.0 - -# Condition -IS ABOVE 6.0 FOR 5 minutes - -# Labels -severity: critical -alertname: High Water Level -station_code: {{ $labels.station_code }} -station_name: {{ $labels.station_name }} - -# Annotations -water_level: {{ $values.water_level }} -threshold: 6.0 -discharge: {{ $values.discharge }} -summary: Critical water level detected at {{ $labels.station_code }} -``` - -### Emergency Water Level Alert -```yaml -Rule Name: emergency-water-level -Query: water_level > 8.0 -Condition: IS ABOVE 8.0 FOR 2 minutes -Labels: - severity: emergency - alertname: Emergency Water Level -Annotations: - threshold: 8.0 - message: IMMEDIATE ACTION REQUIRED - Flood risk imminent -``` - -### Low Water Level Alert -```yaml -Rule Name: low-water-level -Query: water_level < 1.0 -Condition: IS BELOW 1.0 FOR 15 minutes -Labels: - severity: warning - alertname: Low Water Level -Annotations: - threshold: 1.0 - message: Drought conditions detected -``` - -### Data Gap Alert -```yaml -Rule Name: data-gap -Query: - SELECT - station_code, - MAX(timestamp) as last_seen - FROM water_measurements - GROUP BY station_code - HAVING MAX(timestamp) < now() - interval '2 hours' - -Condition: HAS NO DATA FOR 30 minutes -Labels: - severity: warning - alertname: Data Gap - issue: missing-data -``` - -### Rapid Level Change Alert -```yaml -Rule Name: rapid-level-change -Query: - SELECT - station_code, - water_level, - LAG(water_level, 1) OVER (PARTITION BY station_code ORDER BY timestamp) as prev_level - FROM water_measurements - WHERE timestamp > now() - interval '15 minutes' - HAVING ABS(water_level - prev_level) > 0.5 - -Condition: CHANGE > 0.5m FOR 1 minute -Labels: - severity: warning - alertname: Rapid Water Level Change -``` - -## Step 5: Configure Notification Policy - -### Create Notification Policy -```yaml -# Policy Tree -- receiver: matrix-water-alerts - match: - severity: emergency|critical - group_wait: 10s - group_interval: 5m - repeat_interval: 30m - -- receiver: matrix-water-alerts - match: - severity: warning - group_wait: 30s - group_interval: 10m - repeat_interval: 2h -``` - -### Grouping Rules -```yaml -group_by: [alertname, station_code] -group_wait: 10s -group_interval: 5m -repeat_interval: 1h -``` - -## Step 6: Station-Specific Thresholds - -Create separate rules for each station with appropriate thresholds: - -```sql --- P.1 (Chiang Mai) - Urban area, higher thresholds -SELECT * FROM water_measurements -WHERE station_code = 'P.1' AND water_level > 6.5 - --- P.4A (Mae Ping) - Agricultural area -SELECT * FROM water_measurements -WHERE station_code = 'P.4A' AND water_level > 5.0 - --- P.20 (Downstream) - Lower threshold -SELECT * FROM water_measurements -WHERE station_code = 'P.20' AND water_level > 4.0 -``` - -## Step 7: Advanced Features - -### Time-Based Routing -```yaml -# Different receivers for day/night -time_intervals: - - name: working_hours - time_intervals: - - times: - - start_time: '08:00' - end_time: '20:00' - weekdays: ['monday:friday'] - -routes: - - receiver: matrix-alerts-day - match: - severity: warning - active_time_intervals: [working_hours] - - - receiver: matrix-alerts-night - match: - severity: warning - active_time_intervals: ['!working_hours'] -``` - -### Multi-Channel Alerts -```yaml -# Send critical alerts to multiple rooms -- receiver: matrix-emergency - webhook_configs: - - url: https://matrix.org/_matrix/client/v3/rooms/!emergency:matrix.org/send/m.room.message - http_config: - authorization: - credentials: "Bearer EMERGENCY_TOKEN" - - url: https://matrix.org/_matrix/client/v3/rooms/!general:matrix.org/send/m.room.message - http_config: - authorization: - credentials: "Bearer GENERAL_TOKEN" -``` - -## Step 8: Testing - -### Test Contact Point -1. Go to Contact Points in Grafana -2. Select your Matrix contact point -3. Click "Test" button -4. Check Matrix room for test message - -### Test Alert Rules -1. Temporarily lower thresholds -2. Wait for condition to trigger -3. Verify alert appears in Grafana -4. Verify Matrix message received -5. Reset thresholds - -### Manual Alert Trigger -```bash -# Simulate high water level in database -INSERT INTO water_measurements (station_code, water_level, timestamp) -VALUES ('P.1', 7.5, NOW()); -``` - -## Troubleshooting - -### Common Issues - -#### 403 Forbidden -- **Cause**: Invalid Matrix access token -- **Fix**: Regenerate token or check permissions - -#### Room Not Found -- **Cause**: Incorrect room ID format -- **Fix**: Ensure room ID starts with ! and includes homeserver - -#### No Alerts Firing -- **Cause**: Query returns no results -- **Fix**: Test queries in Grafana Explore, check data availability - -#### Alert Spam -- **Cause**: No grouping configured -- **Fix**: Configure proper group_by and intervals - -#### Messages Not Formatted -- **Cause**: Template syntax errors -- **Fix**: Validate JSON template, check Grafana template docs - -### Debug Steps -1. Check Grafana alert rule status -2. Verify contact point test succeeds -3. Check Grafana logs: `/var/log/grafana/grafana.log` -4. Test Matrix API directly with curl -5. Verify database connectivity and query results - -## Environment Variables - -Add to your `.env`: -```bash -MATRIX_HOMESERVER=https://matrix.org -MATRIX_ACCESS_TOKEN=your_access_token_here -MATRIX_ROOM_ID=!your_room_id:matrix.org -GRAFANA_URL=http://your-grafana-host:3000 -``` - -## Example Alert Message -Your Matrix messages will appear as: -``` -🌊 **PING RIVER WATER ALERT** - -**Alert:** High Water Level -**Severity:** CRITICAL -**Station:** P.1 (สถานีเชียงใหม่) - -**Status:** FIRING -**Water Level:** 6.75m -**Threshold:** 6.0m -**Time:** 2025-09-26 14:30:00 -**Discharge:** 450.2 cms - -📈 **Dashboard:** http://grafana:3000 -📍 **Location:** Northern Thailand Ping River -``` - -## Security Notes -- Store Matrix tokens securely (environment variables) -- Use room-specific tokens when possible -- Enable rate limiting to prevent spam -- Consider using dedicated alerting user account -- Regularly rotate access tokens - -This setup provides comprehensive water level monitoring with immediate Matrix notifications when thresholds are exceeded. \ No newline at end of file diff --git a/docs/HTTPS_CONFIGURATION.md b/docs/HTTPS_CONFIGURATION.md deleted file mode 100644 index a05b4d3..0000000 --- a/docs/HTTPS_CONFIGURATION.md +++ /dev/null @@ -1,389 +0,0 @@ -# HTTPS VictoriaMetrics Configuration Guide - -This guide explains how to configure the Thailand Water Monitor to connect to VictoriaMetrics through HTTPS and reverse proxies. - -## Configuration Options - -### 1. Environment Variables for HTTPS - -```bash -# Option 1: Full HTTPS URL (Recommended) -export DB_TYPE=victoriametrics -export VM_HOST=https://vm.example.com -export VM_PORT=443 - -# Option 2: Host and port separately -export DB_TYPE=victoriametrics -export VM_HOST=vm.example.com -export VM_PORT=443 - -# Option 3: Custom port with HTTPS -export DB_TYPE=victoriametrics -export VM_HOST=https://vm.example.com -export VM_PORT=8443 -``` - -### 2. Windows PowerShell Configuration - -```powershell -# Set environment variables for HTTPS -$env:DB_TYPE="victoriametrics" -$env:VM_HOST="https://vm.example.com" -$env:VM_PORT="443" - -# Run the water monitor -python water_scraper_v3.py -``` - -### 3. Linux/Mac Configuration - -```bash -# Set environment variables for HTTPS -export DB_TYPE=victoriametrics -export VM_HOST=https://vm.example.com -export VM_PORT=443 - -# Run the water monitor -python water_scraper_v3.py -``` - -## Reverse Proxy Examples - -### 1. Nginx Reverse Proxy - -```nginx -server { - listen 443 ssl http2; - server_name vm.example.com; - - # SSL Configuration - ssl_certificate /path/to/certificate.crt; - ssl_certificate_key /path/to/private.key; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512; - - # Security headers - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - add_header X-Frame-Options DENY always; - add_header X-Content-Type-Options nosniff always; - - # Optional: Basic authentication - # auth_basic "VictoriaMetrics"; - # auth_basic_user_file /etc/nginx/.htpasswd; - - location / { - proxy_pass http://localhost:8428; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # WebSocket support (if needed) - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - - # Timeouts - proxy_connect_timeout 60s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - } -} - -# Redirect HTTP to HTTPS -server { - listen 80; - server_name vm.example.com; - return 301 https://$server_name$request_uri; -} -``` - -### 2. Apache Reverse Proxy - -```apache - - ServerName vm.example.com - - # SSL Configuration - SSLEngine on - SSLCertificateFile /path/to/certificate.crt - SSLCertificateKeyFile /path/to/private.key - SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 - SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384 - - # Security headers - Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains" - Header always set X-Frame-Options DENY - Header always set X-Content-Type-Options nosniff - - # Reverse proxy configuration - ProxyPreserveHost On - ProxyPass / http://localhost:8428/ - ProxyPassReverse / http://localhost:8428/ - - # Optional: Basic authentication - # AuthType Basic - # AuthName "VictoriaMetrics" - # AuthUserFile /etc/apache2/.htpasswd - # Require valid-user - - - - ServerName vm.example.com - Redirect permanent / https://vm.example.com/ - -``` - -### 3. Traefik Reverse Proxy - -```yaml -# docker-compose.yml with Traefik -version: '3.8' - -services: - traefik: - image: traefik:v2.10 - command: - - --api.dashboard=true - - --entrypoints.web.address=:80 - - --entrypoints.websecure.address=:443 - - --providers.docker=true - - --certificatesresolvers.letsencrypt.acme.tlschallenge=true - - --certificatesresolvers.letsencrypt.acme.email=admin@example.com - - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json - ports: - - "80:80" - - "443:443" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - letsencrypt:/letsencrypt - labels: - - traefik.http.routers.api.rule=Host(`traefik.example.com`) - - traefik.http.routers.api.tls.certresolver=letsencrypt - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - command: - - '--storageDataPath=/victoria-metrics-data' - - '--retentionPeriod=2y' - - '--httpListenAddr=:8428' - volumes: - - vm_data:/victoria-metrics-data - labels: - - traefik.enable=true - - traefik.http.routers.vm.rule=Host(`vm.example.com`) - - traefik.http.routers.vm.tls.certresolver=letsencrypt - - traefik.http.services.vm.loadbalancer.server.port=8428 - -volumes: - vm_data: - letsencrypt: -``` - -## Testing HTTPS Configuration - -### 1. Test Connection - -```bash -# Test HTTPS connection -curl -k https://vm.example.com/health - -# Test with specific port -curl -k https://vm.example.com:8443/health - -# Test API endpoint -curl -k "https://vm.example.com/api/v1/query?query=up" -``` - -### 2. Test with Water Monitor - -```bash -# Set environment variables -export DB_TYPE=victoriametrics -export VM_HOST=https://vm.example.com -export VM_PORT=443 - -# Test with demo script -python demo_databases.py victoriametrics - -# Run full water monitor -python water_scraper_v3.py -``` - -### 3. Verify SSL Certificate - -```bash -# Check SSL certificate -openssl s_client -connect vm.example.com:443 -servername vm.example.com - -# Check certificate expiration -echo | openssl s_client -connect vm.example.com:443 2>/dev/null | openssl x509 -noout -dates -``` - -## Configuration Examples - -### 1. Production HTTPS Setup - -```bash -# Environment variables for production -export DB_TYPE=victoriametrics -export VM_HOST=https://metrics.company.com -export VM_PORT=443 -export LOG_LEVEL=INFO -export SCRAPING_INTERVAL_HOURS=1 - -# Run water monitor -python water_scraper_v3.py -``` - -### 2. Development with Self-Signed Certificate - -```bash -# For development with self-signed certificates -export DB_TYPE=victoriametrics -export VM_HOST=https://dev-vm.local -export VM_PORT=443 -export PYTHONHTTPSVERIFY=0 # Disable SSL verification (dev only) - -python water_scraper_v3.py -``` - -### 3. Custom Port Configuration - -```bash -# Custom HTTPS port -export DB_TYPE=victoriametrics -export VM_HOST=https://vm.example.com -export VM_PORT=8443 - -python water_scraper_v3.py -``` - -## Troubleshooting HTTPS Issues - -### 1. SSL Certificate Errors - -```bash -# Error: SSL certificate verify failed -# Solution: Check certificate validity -openssl x509 -in certificate.crt -text -noout - -# Temporary workaround (not recommended for production) -export PYTHONHTTPSVERIFY=0 -``` - -### 2. Connection Timeout - -```bash -# Error: Connection timeout -# Check firewall and network connectivity -telnet vm.example.com 443 -nc -zv vm.example.com 443 -``` - -### 3. DNS Resolution Issues - -```bash -# Error: Name resolution failed -# Check DNS resolution -nslookup vm.example.com -dig vm.example.com -``` - -### 4. Proxy Configuration Issues - -```bash -# Check proxy logs -# Nginx -tail -f /var/log/nginx/error.log - -# Apache -tail -f /var/log/apache2/error.log - -# Test direct connection to backend -curl http://localhost:8428/health -``` - -## Security Best Practices - -### 1. SSL/TLS Configuration - -- Use TLS 1.2 or higher -- Disable weak ciphers -- Enable HSTS headers -- Use strong SSL certificates - -### 2. Authentication - -```nginx -# Basic authentication in Nginx -auth_basic "VictoriaMetrics Access"; -auth_basic_user_file /etc/nginx/.htpasswd; - -# Create password file -htpasswd -c /etc/nginx/.htpasswd username -``` - -### 3. Network Security - -- Use firewall rules to restrict access -- Consider VPN for internal access -- Implement rate limiting -- Monitor access logs - -### 4. Certificate Management - -```bash -# Auto-renewal with Let's Encrypt -certbot renew --dry-run - -# Certificate monitoring -echo | openssl s_client -connect vm.example.com:443 2>/dev/null | \ -openssl x509 -noout -dates | grep notAfter -``` - -## Docker Configuration for HTTPS - -### 1. Docker Compose with HTTPS - -```yaml -version: '3.8' - -services: - water-monitor: - build: . - environment: - - DB_TYPE=victoriametrics - - VM_HOST=https://vm.example.com - - VM_PORT=443 - restart: unless-stopped - depends_on: - - victoriametrics - - victoriametrics: - image: victoriametrics/victoria-metrics:latest - ports: - - "8428:8428" - volumes: - - vm_data:/victoria-metrics-data - command: - - '--storageDataPath=/victoria-metrics-data' - - '--retentionPeriod=2y' - - '--httpListenAddr=:8428' - -volumes: - vm_data: -``` - -### 2. Environment File (.env) - -```bash -# .env file -DB_TYPE=victoriametrics -VM_HOST=https://vm.example.com -VM_PORT=443 -LOG_LEVEL=INFO -SCRAPING_INTERVAL_HOURS=1 -``` - -This configuration guide provides comprehensive instructions for setting up HTTPS connectivity to VictoriaMetrics through reverse proxies, ensuring secure and reliable data transmission for the Thailand Water Monitor. diff --git a/docs/MIGRATION_QUICKSTART.md b/docs/MIGRATION_QUICKSTART.md deleted file mode 100644 index 20f37e6..0000000 --- a/docs/MIGRATION_QUICKSTART.md +++ /dev/null @@ -1,136 +0,0 @@ -# Geolocation Migration Quick Start - -This is a quick reference guide for updating a running Thailand Water Monitor system to add geolocation support for Grafana geomap. - -## 🚀 **Quick Migration (5 minutes)** - -### **Step 1: Stop Application** -```bash -# Stop the service (choose your method) -sudo systemctl stop water-monitor -# OR -docker stop water-monitor -# OR use Ctrl+C if running manually -``` - -### **Step 2: Backup Database** -```bash -# SQLite backup -cp water_monitoring.db water_monitoring.db.backup - -# PostgreSQL backup -pg_dump water_monitoring > backup.sql - -# MySQL backup -mysqldump water_monitoring > backup.sql -``` - -### **Step 3: Run Migration** -```bash -# Run the automated migration script -python migrate_geolocation.py -``` - -### **Step 4: Restart Application** -```bash -# Restart the service -sudo systemctl start water-monitor -# OR -docker start water-monitor -# OR -python water_scraper_v3.py -``` - -## ✅ **Expected Output** -``` -2025-07-28 17:30:00,123 - INFO - Starting geolocation column migration... -2025-07-28 17:30:00,124 - INFO - Detected database type: SQLITE -2025-07-28 17:30:00,127 - INFO - Added latitude column -2025-07-28 17:30:00,128 - INFO - Added longitude column -2025-07-28 17:30:00,129 - INFO - Added geohash column -2025-07-28 17:30:00,133 - INFO - ✅ Migration completed successfully! -``` - -## 🗺️ **Verify Geolocation Works** - -### **Check Database** -```bash -# SQLite -sqlite3 water_monitoring.db "SELECT station_code, latitude, longitude, geohash FROM stations WHERE station_code = 'P.1';" - -# Expected output: P.1|15.6944|100.2028|w5q6uuhvfcfp25 -``` - -### **Test Application** -```bash -# Run a test cycle -python water_scraper_v3.py --test - -# Should complete without errors -``` - -## 🔧 **Grafana Setup** - -### **Query for Geomap** -```sql -SELECT - s.latitude, s.longitude, s.station_code, s.english_name, - m.water_level, m.discharge_percent -FROM stations s -JOIN water_measurements m ON s.id = m.station_id -WHERE s.latitude IS NOT NULL - AND m.timestamp = (SELECT MAX(timestamp) FROM water_measurements WHERE station_id = s.id) -``` - -### **Geomap Configuration** -1. Create new panel → Select "Geomap" -2. Set **Latitude field**: `latitude` -3. Set **Longitude field**: `longitude` -4. Set **Color field**: `water_level` -5. Set **Size field**: `discharge_percent` - -## 🚨 **Troubleshooting** - -### **Database Locked** -```bash -sudo systemctl stop water-monitor -pkill -f water_scraper -sleep 5 -python migrate_geolocation.py -``` - -### **Permission Error** -```bash -sudo chown $USER:$USER water_monitoring.db -chmod 664 water_monitoring.db -``` - -### **Missing Dependencies** -```bash -pip install psycopg2-binary pymysql -``` - -## 🔄 **Rollback (if needed)** -```bash -# Stop application -sudo systemctl stop water-monitor - -# Restore backup -cp water_monitoring.db.backup water_monitoring.db - -# Restart -sudo systemctl start water-monitor -``` - -## 📚 **More Information** -- **Full Guide**: See `GEOLOCATION_GUIDE.md` -- **Migration Script**: `migrate_geolocation.py` -- **Database Schema**: Updated with latitude, longitude, geohash columns - -## 🎯 **What You Get** -- ✅ **P.1 Station** ready for geomap (Nawarat Bridge) -- ✅ **Database Schema** updated for all 16 stations -- ✅ **Grafana Compatible** data structure -- ✅ **Backward Compatible** - existing data preserved - -**Total Time**: ~5 minutes for complete migration diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md deleted file mode 100644 index c1ef117..0000000 --- a/docs/PROJECT_STATUS.md +++ /dev/null @@ -1,206 +0,0 @@ -# Thailand Water Monitor - Current Project Status - -## 📁 **Clean Project Structure** - -The project has been cleaned up and organized with the following structure: - -``` -water_level_monitor/ -├── 📄 .gitignore # Git ignore rules -├── 📄 README.md # Main project documentation -├── 📄 requirements.txt # Python dependencies -├── 📄 config.py # Configuration management -├── 📄 water_scraper_v3.py # Main application (15-min scheduler) -├── 📄 database_adapters.py # Multi-database support -├── 📄 demo_databases.py # Database demonstration -├── 📄 Dockerfile # Container configuration -├── 📄 docker-compose.victoriametrics.yml # VictoriaMetrics stack -├── 📚 Documentation/ -│ ├── 📄 DATABASE_DEPLOYMENT_GUIDE.md # Multi-database setup guide -│ ├── 📄 DEBIAN_TROUBLESHOOTING.md # Linux deployment guide -│ ├── 📄 ENHANCED_SCHEDULER_GUIDE.md # 15-minute scheduler guide -│ ├── 📄 GAP_FILLING_GUIDE.md # Data gap filling guide -│ ├── 📄 HTTPS_CONFIGURATION.md # HTTPS setup guide -│ └── 📄 VICTORIAMETRICS_SETUP.md # VictoriaMetrics guide -└── 📁 grafana/ # Grafana configuration - ├── 📁 provisioning/ - │ ├── 📁 datasources/ - │ │ └── 📄 victoriametrics.yml # VictoriaMetrics data source - │ └── 📁 dashboards/ - │ └── 📄 dashboard.yml # Dashboard provider config - └── 📁 dashboards/ - └── 📄 water-monitoring-dashboard.json # Pre-built dashboard -``` - -## 🧹 **Files Removed During Cleanup** - -### **Old Data Files** -- ❌ `thailand_water_data_v2.csv` - Old CSV export -- ❌ `water_monitor.log` - Log file (regenerated automatically) -- ❌ `water_monitoring.db` - SQLite database (recreated automatically) - -### **Outdated Documentation** -- ❌ `FINAL_SUMMARY.md` - Contained references to non-existent v2 files -- ❌ `PROJECT_SUMMARY.md` - Outdated project information - -### **System Files** -- ❌ `__pycache__/` - Python compiled files directory - -## ✅ **Current Features** - -### **Enhanced 15-Minute Scheduler** -- **Timing**: Runs every 15 minutes (1:00, 1:15, 1:30, 1:45, 2:00, etc.) -- **Full Checks**: At :00 minutes (gap filling + data updates) -- **Quick Checks**: At :15, :30, :45 minutes (data fetch only) -- **Gap Filling**: Automatically fills missing historical data -- **Data Updates**: Updates existing records when values change - -### **Multi-Database Support** -- **VictoriaMetrics** (Recommended) - High-performance time-series -- **InfluxDB** - Purpose-built time-series database -- **PostgreSQL + TimescaleDB** - Relational with time-series optimization -- **MySQL** - Traditional relational database -- **SQLite** - Local development and testing - -### **Production Features** -- **Docker Support**: Complete containerization -- **Grafana Integration**: Pre-built dashboards -- **HTTPS Configuration**: Secure deployment options -- **Health Monitoring**: Comprehensive logging and error handling -- **Gap Detection**: Automatic identification of missing data -- **Retry Logic**: Database lock handling and network error recovery - -## 🚀 **Quick Start** - -### **1. Basic Setup (SQLite)** -```bash -cd water_level_monitor -pip install -r requirements.txt -python water_scraper_v3.py -``` - -### **2. VictoriaMetrics Setup** -```bash -# Start VictoriaMetrics + Grafana -docker-compose -f docker-compose.victoriametrics.yml up -d - -# Configure environment -export DB_TYPE=victoriametrics -export VM_HOST=localhost -export VM_PORT=8428 - -# Run monitor -python water_scraper_v3.py -``` - -### **3. Test Different Databases** -```bash -# Test all supported databases -python demo_databases.py all - -# Test specific database -python demo_databases.py victoriametrics -``` - -## 📊 **Data Collection** - -### **Station Coverage** -- **16 Water Monitoring Stations** across Thailand -- **Accurate Station Codes**: P.1, P.20, P.21, P.4A, P.5, P.67, P.75, P.76, P.77, P.81, P.82, P.84, P.85, P.87, P.92, P.103 -- **Bilingual Names**: Thai and English station identification - -### **Metrics Collected** -- 🌊 **Water Level**: Measured in meters (m) -- 💧 **Discharge**: Measured in cubic meters per second (cms) -- 📊 **Discharge Percentage**: Relative to station capacity -- ⏰ **Timestamp**: Hour 24 handling (midnight = 00:00 next day) - -### **Data Frequency** -- **Every 15 Minutes**: Continuous monitoring -- **~300+ Data Points**: Per collection cycle -- **Automatic Gap Filling**: Historical data recovery -- **Data Updates**: Changed values detection and correction - -## 🔧 **Command Line Tools** - -### **Main Application** -```bash -python water_scraper_v3.py # Run continuous monitoring -python water_scraper_v3.py --test # Single test cycle -python water_scraper_v3.py --help # Show help -``` - -### **Gap Management** -```bash -python water_scraper_v3.py --check-gaps [days] # Check for missing data -python water_scraper_v3.py --fill-gaps [days] # Fill missing data gaps -python water_scraper_v3.py --update-data [days] # Update existing data -``` - -### **Database Testing** -```bash -python demo_databases.py # SQLite demo -python demo_databases.py victoriametrics # VictoriaMetrics demo -python demo_databases.py all # Test all databases -``` - -## 📈 **Monitoring & Visualization** - -### **Grafana Dashboard** -- **URL**: http://localhost:3000 (when using docker-compose) -- **Username**: admin -- **Password**: admin_password -- **Features**: Time series charts, status tables, gauges, alerts - -### **VictoriaMetrics API** -- **URL**: http://localhost:8428 -- **Health**: http://localhost:8428/health -- **Metrics**: http://localhost:8428/metrics -- **Query API**: http://localhost:8428/api/v1/query - -## 🛡️ **Security & Production** - -### **HTTPS Configuration** -- Complete guide in `HTTPS_CONFIGURATION.md` -- SSL certificate setup -- Reverse proxy configuration -- Security best practices - -### **Deployment Options** -- **Docker**: Containerized deployment -- **Systemd**: Linux service configuration -- **Cloud**: AWS, GCP, Azure deployment guides -- **Monitoring**: Health checks and alerting - -## 📚 **Documentation** - -### **Available Guides** -1. **README.md** - Main project documentation -2. **DATABASE_DEPLOYMENT_GUIDE.md** - Multi-database setup -3. **ENHANCED_SCHEDULER_GUIDE.md** - 15-minute scheduler details -4. **GAP_FILLING_GUIDE.md** - Data integrity and gap filling -5. **DEBIAN_TROUBLESHOOTING.md** - Linux deployment troubleshooting -6. **VICTORIAMETRICS_SETUP.md** - VictoriaMetrics configuration -7. **HTTPS_CONFIGURATION.md** - Secure deployment setup - -### **Key Features Documented** -- ✅ Installation and configuration -- ✅ Multi-database support -- ✅ 15-minute scheduling system -- ✅ Gap filling and data integrity -- ✅ Production deployment -- ✅ Monitoring and troubleshooting -- ✅ Security configuration - -## 🎯 **Project Status: PRODUCTION READY** - -The Thailand Water Monitor is now: -- ✅ **Clean**: All old and redundant files removed -- ✅ **Organized**: Clear project structure with proper documentation -- ✅ **Enhanced**: 15-minute scheduling with gap filling -- ✅ **Scalable**: Multi-database support with VictoriaMetrics -- ✅ **Secure**: HTTPS configuration and security best practices -- ✅ **Monitored**: Comprehensive logging and Grafana dashboards -- ✅ **Documented**: Complete guides for all features and deployment options - -The project is ready for production deployment with professional-grade monitoring capabilities. diff --git a/docs/PROJECT_STRUCTURE.md b/docs/PROJECT_STRUCTURE.md deleted file mode 100644 index 06f724b..0000000 --- a/docs/PROJECT_STRUCTURE.md +++ /dev/null @@ -1,272 +0,0 @@ -# 🏗️ Project Structure - Northern Thailand Ping River Monitor - -## 📁 Directory Layout - -``` -Northern-Thailand-Ping-River-Monitor/ -├── 📁 src/ # Main application source code -│ ├── __init__.py # Package initialization -│ ├── main.py # CLI entry point and main application -│ ├── water_scraper_v3.py # Core data collection engine -│ ├── web_api.py # FastAPI web interface -│ ├── config.py # Configuration management -│ ├── database_adapters.py # Database abstraction layer -│ ├── models.py # Data models and type definitions -│ ├── exceptions.py # Custom exception classes -│ ├── validators.py # Data validation layer -│ ├── metrics.py # Metrics collection system -│ ├── health_check.py # Health monitoring system -│ ├── rate_limiter.py # Rate limiting and request tracking -│ └── logging_config.py # Enhanced logging configuration -├── 📁 docs/ # Documentation files -│ ├── STATION_MANAGEMENT_GUIDE.md # Station management documentation -│ ├── ENHANCEMENT_SUMMARY.md # Feature enhancement summary -│ └── PROJECT_STRUCTURE.md # This file -├── 📁 scripts/ # Utility scripts -│ └── migrate_geolocation.py # Database migration script -├── 📁 grafana/ # Grafana configuration -│ ├── dashboards/ # Dashboard definitions -│ └── provisioning/ # Grafana provisioning config -├── 📁 tests/ # Test files -│ ├── test_integration.py # Integration test suite -│ ├── test_station_management.py # Station management tests -│ └── test_api.py # API endpoint tests -├── 📄 run.py # Simple startup script -├── 📄 requirements.txt # Production dependencies -├── 📄 requirements-dev.txt # Development dependencies -├── 📄 setup.py # Package installation script -├── 📄 Dockerfile # Docker container definition -├── 📄 docker-compose.victoriametrics.yml # Complete stack deployment -├── 📄 Makefile # Common development tasks -├── 📄 .env.example # Environment configuration template -├── 📄 .gitignore # Git ignore patterns -├── 📄 .gitlab-ci.yml # CI/CD pipeline configuration -├── 📄 LICENSE # MIT license -├── 📄 README.md # Main project documentation -└── 📄 CONTRIBUTING.md # Contribution guidelines -``` - -## 🔧 Core Components - -### **Application Layer** -- **`src/main.py`** - Command-line interface and application orchestration -- **`src/web_api.py`** - FastAPI web interface with REST endpoints -- **`src/water_scraper_v3.py`** - Core data collection and processing engine - -### **Data Layer** -- **`src/database_adapters.py`** - Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics) -- **`src/models.py`** - Pydantic data models and type definitions -- **`src/validators.py`** - Data validation and sanitization - -### **Infrastructure Layer** -- **`src/config.py`** - Configuration management with environment variable support -- **`src/logging_config.py`** - Structured logging with rotation and colors -- **`src/metrics.py`** - Application metrics collection (counters, gauges, histograms) -- **`src/health_check.py`** - System health monitoring and status checks - -### **Utility Layer** -- **`src/exceptions.py`** - Custom exception hierarchy -- **`src/rate_limiter.py`** - API rate limiting and request tracking - -## 🌐 Web API Structure - -### **Endpoints Organization** -``` -/ # Dashboard homepage -├── /health # System health status -├── /metrics # Application metrics -├── /config # Configuration (masked) -├── /stations # Station management -│ ├── GET / # List all stations -│ ├── POST / # Create new station -│ ├── GET /{id} # Get specific station -│ ├── PUT /{id} # Update station -│ └── DELETE /{id} # Delete station -├── /measurements # Data access -│ ├── /latest # Latest measurements -│ └── /station/{code} # Station-specific data -└── /scraping # Data collection control - ├── /trigger # Manual data collection - └── /status # Scraping status -``` - -### **API Models** -- **Request Models**: Station creation/update, query parameters -- **Response Models**: Station info, measurements, health status -- **Error Models**: Standardized error responses - -## 🗄️ Database Architecture - -### **Supported Databases** -1. **SQLite** - Local development and testing -2. **MySQL** - Traditional relational database -3. **PostgreSQL** - Advanced relational with TimescaleDB support -4. **InfluxDB** - Purpose-built time-series database -5. **VictoriaMetrics** - High-performance metrics storage - -### **Schema Design** -```sql --- Stations table -stations ( - id INTEGER PRIMARY KEY, - station_code VARCHAR(10) UNIQUE, - thai_name VARCHAR(255), - english_name VARCHAR(255), - latitude DECIMAL(10,8), - longitude DECIMAL(11,8), - geohash VARCHAR(20), - status VARCHAR(20), - created_at TIMESTAMP, - updated_at TIMESTAMP -) - --- Measurements table -water_measurements ( - id BIGINT PRIMARY KEY, - timestamp DATETIME, - station_id INTEGER, - water_level DECIMAL(10,3), - discharge DECIMAL(10,2), - discharge_percent DECIMAL(5,2), - status VARCHAR(20), - created_at TIMESTAMP, - FOREIGN KEY (station_id) REFERENCES stations(id), - UNIQUE(timestamp, station_id) -) -``` - -## 🐳 Docker Architecture - -### **Multi-Stage Build** -1. **Builder Stage** - Compile dependencies and build artifacts -2. **Production Stage** - Minimal runtime environment - -### **Service Composition** -- **ping-river-monitor** - Data collection service -- **ping-river-api** - Web API service -- **victoriametrics** - Time-series database -- **grafana** - Visualization dashboard - -## 📊 Monitoring Architecture - -### **Metrics Collection** -- **Counters** - API requests, database operations, scraping cycles -- **Gauges** - Current values, connection status, resource usage -- **Histograms** - Response times, processing durations - -### **Health Checks** -- **Database Health** - Connection status, data freshness -- **API Health** - External API availability, response times -- **System Health** - Memory usage, disk space, CPU load - -### **Logging Levels** -- **DEBUG** - Detailed execution information -- **INFO** - General operational messages -- **WARNING** - Potential issues and recoverable errors -- **ERROR** - Serious problems requiring attention -- **CRITICAL** - System-threatening issues - -## 🔧 Configuration Management - -### **Environment Variables** -```bash -# Database -DB_TYPE=victoriametrics -VM_HOST=localhost -VM_PORT=8428 - -# Application -SCRAPING_INTERVAL_HOURS=1 -LOG_LEVEL=INFO -DATA_RETENTION_DAYS=365 - -# Security -SECRET_KEY=your-secret-key -API_KEY=your-api-key -``` - -### **Configuration Hierarchy** -1. Environment variables (highest priority) -2. .env file -3. Default values in config.py (lowest priority) - -## 🧪 Testing Architecture - -### **Test Categories** -- **Unit Tests** - Individual component testing -- **Integration Tests** - System component interaction -- **API Tests** - Endpoint functionality and responses -- **Performance Tests** - Load and stress testing - -### **Test Data** -- **Mock Data** - Simulated API responses -- **Test Database** - Isolated test environment -- **Fixtures** - Reusable test data sets - -## 📦 Deployment Architecture - -### **Development** -```bash -python run.py --web-api # Local development server -``` - -### **Production** -```bash -docker-compose up -d # Full stack deployment -``` - -### **CI/CD Pipeline** -1. **Test Stage** - Run all tests and quality checks -2. **Build Stage** - Create Docker images -3. **Deploy Stage** - Deploy to staging/production -4. **Health Check** - Verify deployment success - -## 🔒 Security Architecture - -### **Input Validation** -- Pydantic models for API requests -- Data range validation for measurements -- SQL injection prevention through ORM - -### **Authentication** (Future) -- API key authentication -- JWT token support -- Role-based access control - -### **Data Protection** -- Environment variable configuration -- Sensitive data masking in logs -- HTTPS support for production - -## 📈 Performance Architecture - -### **Optimization Strategies** -- Database connection pooling -- Query optimization and indexing -- Response caching for static data -- Async processing for I/O operations - -### **Scalability Considerations** -- Horizontal scaling with load balancers -- Database read replicas -- Microservice architecture readiness -- Container orchestration support - -## 🔄 Data Flow Architecture - -### **Collection Flow** -``` -External API → Rate Limiter → Data Validator → Database Adapter → Database -``` - -### **API Flow** -``` -HTTP Request → FastAPI → Business Logic → Database Adapter → HTTP Response -``` - -### **Monitoring Flow** -``` -Application Events → Metrics Collector → Health Checks → Monitoring Dashboard -``` - -This architecture provides a solid foundation for a production-ready water monitoring system with excellent maintainability, scalability, and observability. \ No newline at end of file diff --git a/ping-river-monitor.spec b/ping-river-monitor.spec deleted file mode 100644 index 00306e5..0000000 --- a/ping-river-monitor.spec +++ /dev/null @@ -1,38 +0,0 @@ -# -*- mode: python ; coding: utf-8 -*- - - -a = Analysis( - ['run.py'], - pathex=[], - binaries=[], - datas=[('.env', '.'), ('sql', 'sql'), ('README.md', '.'), ('POSTGRESQL_SETUP.md', '.'), ('SQLITE_MIGRATION.md', '.')], - hiddenimports=['psycopg2', 'sqlalchemy.dialects.postgresql', 'sqlalchemy.dialects.sqlite', 'dotenv', 'pydantic', 'fastapi', 'uvicorn', 'schedule', 'pandas'], - hookspath=[], - hooksconfig={}, - runtime_hooks=[], - excludes=[], - noarchive=False, - optimize=0, -) -pyz = PYZ(a.pure) - -exe = EXE( - pyz, - a.scripts, - a.binaries, - a.datas, - [], - name='ping-river-monitor', - debug=False, - bootloader_ignore_signals=False, - strip=False, - upx=True, - upx_exclude=[], - runtime_tmpdir=None, - console=True, - disable_windowed_traceback=False, - argv_emulation=False, - target_arch=None, - codesign_identity=None, - entitlements_file=None, -) diff --git a/scripts/encode_password.py b/scripts/encode_password.py deleted file mode 100644 index 45e55e7..0000000 --- a/scripts/encode_password.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -""" -Password URL encoder for PostgreSQL connection strings -""" - -import urllib.parse -import sys - -def encode_password(password: str) -> str: - """URL encode a password for use in connection strings""" - return urllib.parse.quote(password, safe='') - -def build_connection_string(username: str, password: str, host: str, port: int, database: str) -> str: - """Build a properly encoded PostgreSQL connection string""" - encoded_password = encode_password(password) - return f"postgresql://{username}:{encoded_password}@{host}:{port}/{database}" - -def main(): - print("PostgreSQL Password URL Encoder") - print("=" * 40) - - if len(sys.argv) > 1: - # Password provided as argument - password = sys.argv[1] - else: - # Interactive mode - password = input("Enter your password: ") - - encoded = encode_password(password) - - print(f"\nOriginal password: {password}") - print(f"URL encoded: {encoded}") - - # Optional: build full connection string - try: - build_full = input("\nBuild full connection string? (y/N): ").strip().lower() == 'y' - except (EOFError, KeyboardInterrupt): - print("\nDone!") - return - - if build_full: - username = input("Username: ").strip() - host = input("Host: ").strip() - port = input("Port [5432]: ").strip() or "5432" - database = input("Database [water_monitoring]: ").strip() or "water_monitoring" - - connection_string = build_connection_string(username, password, host, int(port), database) - - print(f"\nComplete connection string:") - print(f"POSTGRES_CONNECTION_STRING={connection_string}") - - print(f"\nAdd this to your .env file:") - print(f"DB_TYPE=postgresql") - print(f"POSTGRES_CONNECTION_STRING={connection_string}") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/generate_badges.py b/scripts/generate_badges.py deleted file mode 100644 index d09f829..0000000 --- a/scripts/generate_badges.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate status badges for README.md -""" - -import json -import requests -from datetime import datetime - -def generate_badge_url(label, message, color="brightgreen"): - """Generate a shields.io badge URL""" - return f"https://img.shields.io/badge/{label}-{message}-{color}" - -def generate_workflow_badge(repo_url, workflow_name, branch="main"): - """Generate workflow status badge""" - # For Gitea, you might need to adjust this based on your instance - badge_url = f"{repo_url}/actions/workflows/{workflow_name}/badge.svg?branch={branch}" - return badge_url - -def main(): - """Generate badges for the project""" - repo_url = "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor" - - badges = { - "CI/CD": generate_workflow_badge(repo_url, "ci.yml"), - "Security": generate_workflow_badge(repo_url, "security.yml"), - "Documentation": generate_workflow_badge(repo_url, "docs.yml"), - "Python": generate_badge_url("Python", "3.9%2B", "blue"), - "FastAPI": generate_badge_url("FastAPI", "0.104%2B", "green"), - "Docker": generate_badge_url("Docker", "Ready", "blue"), - "License": generate_badge_url("License", "MIT", "green"), - "Version": generate_badge_url("Version", "v3.1.3", "blue"), - } - - print("# Status Badges") - print() - print("Add these badges to your README.md:") - print() - - for name, url in badges.items(): - print(f"[![{name}]({url})]({repo_url})") - - print() - print("# Markdown Format") - print() - - badge_line = " ".join([f"[![{name}]({url})]({repo_url})" for name, url in badges.items()]) - print(badge_line) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/init_git.bat b/scripts/init_git.bat deleted file mode 100644 index e07b9e6..0000000 --- a/scripts/init_git.bat +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -REM Git initialization script for Northern Thailand Ping River Monitor - -echo 🏔️ Initializing Git repository for Northern Thailand Ping River Monitor - -REM Initialize git repository -git init - -REM Add remote origin -git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git - -REM Add all files -git add . - -REM Initial commit -git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.3 - -Features: -- Real-time water level monitoring for Ping River Basin -- 16 monitoring stations from Chiang Dao to Nakhon Sawan -- FastAPI web interface with station management -- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics) -- Comprehensive monitoring and health checks -- Docker deployment with Grafana integration -- Production-ready architecture with CI/CD pipeline" - -echo ✅ Git repository initialized successfully! -echo. -echo Next steps: -echo 1. Review and edit .env file with your configuration -echo 2. Push to remote repository: -echo git push -u origin main -echo. -echo 3. Start the application: -echo python run.py --web-api \ No newline at end of file diff --git a/scripts/init_git.sh b/scripts/init_git.sh deleted file mode 100644 index e9d881b..0000000 --- a/scripts/init_git.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# Git initialization script for Northern Thailand Ping River Monitor - -echo "🏔️ Initializing Git repository for Northern Thailand Ping River Monitor" - -# Initialize git repository -git init - -# Add remote origin -git remote add origin https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor.git - -# Create .gitignore if it doesn't exist -if [ ! -f .gitignore ]; then - echo "Creating .gitignore file..." - cat > .gitignore << 'EOF' -# Python -__pycache__/ -*.py[cod] -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# Virtual environments -.env -.venv -env/ -venv/ -ENV/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo - -# Logs -*.log -logs/ - -# Database files -*.db -*.sqlite -*.sqlite3 - -# OS -.DS_Store -Thumbs.db -EOF -fi - -# Add all files -git add . - -# Initial commit -git commit -m "Initial commit: Northern Thailand Ping River Monitor v3.1.3 - -Features: -- Real-time water level monitoring for Ping River Basin -- 16 monitoring stations from Chiang Dao to Nakhon Sawan -- FastAPI web interface with station management -- Multi-database support (SQLite, MySQL, PostgreSQL, InfluxDB, VictoriaMetrics) -- Comprehensive monitoring and health checks -- Docker deployment with Grafana integration -- Production-ready architecture with CI/CD pipeline" - -echo "✅ Git repository initialized successfully!" -echo "" -echo "Next steps:" -echo "1. Review and edit .env file with your configuration" -echo "2. Push to remote repository:" -echo " git push -u origin main" -echo "" -echo "3. Start the application:" -echo " make run-api" -echo " # or: python run.py --web-api" \ No newline at end of file diff --git a/scripts/migrate_geolocation.py b/scripts/migrate_geolocation.py deleted file mode 100644 index 10bfc6d..0000000 --- a/scripts/migrate_geolocation.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python3 -""" -Migration script to add geolocation columns to existing water monitoring database -""" - -import os -import sys -import sqlite3 -import logging -from typing import Dict, Any - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) - -def migrate_sqlite(db_path: str = 'water_monitoring.db') -> bool: - """Migrate SQLite database to add geolocation columns""" - try: - logging.info(f"Migrating SQLite database: {db_path}") - - # Connect to database - conn = sqlite3.connect(db_path) - cursor = conn.cursor() - - # Check if columns already exist - cursor.execute("PRAGMA table_info(stations)") - columns = [column[1] for column in cursor.fetchall()] - - logging.info(f"Current columns in stations table: {columns}") - - # Add columns if they don't exist - columns_added = [] - - if 'latitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN latitude REAL") - columns_added.append('latitude') - logging.info("Added latitude column") - - if 'longitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN longitude REAL") - columns_added.append('longitude') - logging.info("Added longitude column") - - if 'geohash' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN geohash TEXT") - columns_added.append('geohash') - logging.info("Added geohash column") - - if columns_added: - # Update P.1 station with sample geolocation data - cursor.execute(""" - UPDATE stations - SET latitude = 15.6944, longitude = 100.2028, geohash = 'w5q6uuhvfcfp25' - WHERE station_code = 'P.1' - """) - - # Commit changes - conn.commit() - logging.info(f"Successfully added columns: {', '.join(columns_added)}") - logging.info("Updated P.1 station with sample geolocation data") - else: - logging.info("All geolocation columns already exist") - - # Verify the changes - cursor.execute("SELECT station_code, latitude, longitude, geohash FROM stations WHERE station_code = 'P.1'") - result = cursor.fetchone() - if result: - logging.info(f"P.1 station geolocation: {result}") - - conn.close() - return True - - except Exception as e: - logging.error(f"Error migrating SQLite database: {e}") - return False - -def migrate_postgresql(connection_string: str) -> bool: - """Migrate PostgreSQL database to add geolocation columns""" - try: - import psycopg2 - from urllib.parse import urlparse - - logging.info("Migrating PostgreSQL database") - - # Parse connection string - parsed = urlparse(connection_string) - - # Connect to database - conn = psycopg2.connect( - host=parsed.hostname, - port=parsed.port or 5432, - database=parsed.path[1:], # Remove leading slash - user=parsed.username, - password=parsed.password - ) - cursor = conn.cursor() - - # Check if columns exist - cursor.execute(""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = 'stations' - """) - columns = [row[0] for row in cursor.fetchall()] - - logging.info(f"Current columns in stations table: {columns}") - - # Add columns if they don't exist - columns_added = [] - - if 'latitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN latitude DECIMAL(10,8)") - columns_added.append('latitude') - logging.info("Added latitude column") - - if 'longitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN longitude DECIMAL(11,8)") - columns_added.append('longitude') - logging.info("Added longitude column") - - if 'geohash' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN geohash VARCHAR(20)") - columns_added.append('geohash') - logging.info("Added geohash column") - - if columns_added: - # Update P.1 station with sample geolocation data - cursor.execute(""" - UPDATE stations - SET latitude = 15.6944, longitude = 100.2028, geohash = 'w5q6uuhvfcfp25' - WHERE station_code = 'P.1' - """) - - # Commit changes - conn.commit() - logging.info(f"Successfully added columns: {', '.join(columns_added)}") - logging.info("Updated P.1 station with sample geolocation data") - else: - logging.info("All geolocation columns already exist") - - conn.close() - return True - - except ImportError: - logging.error("psycopg2 not installed. Run: pip install psycopg2-binary") - return False - except Exception as e: - logging.error(f"Error migrating PostgreSQL database: {e}") - return False - -def migrate_mysql(connection_string: str) -> bool: - """Migrate MySQL database to add geolocation columns""" - try: - import pymysql - from urllib.parse import urlparse - - logging.info("Migrating MySQL database") - - # Parse connection string - parsed = urlparse(connection_string) - - # Connect to database - conn = pymysql.connect( - host=parsed.hostname, - port=parsed.port or 3306, - database=parsed.path[1:], # Remove leading slash - user=parsed.username, - password=parsed.password - ) - cursor = conn.cursor() - - # Check if columns exist - cursor.execute("DESCRIBE stations") - columns = [row[0] for row in cursor.fetchall()] - - logging.info(f"Current columns in stations table: {columns}") - - # Add columns if they don't exist - columns_added = [] - - if 'latitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN latitude DECIMAL(10,8)") - columns_added.append('latitude') - logging.info("Added latitude column") - - if 'longitude' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN longitude DECIMAL(11,8)") - columns_added.append('longitude') - logging.info("Added longitude column") - - if 'geohash' not in columns: - cursor.execute("ALTER TABLE stations ADD COLUMN geohash VARCHAR(20)") - columns_added.append('geohash') - logging.info("Added geohash column") - - if columns_added: - # Update P.1 station with sample geolocation data - cursor.execute(""" - UPDATE stations - SET latitude = 15.6944, longitude = 100.2028, geohash = 'w5q6uuhvfcfp25' - WHERE station_code = 'P.1' - """) - - # Commit changes - conn.commit() - logging.info(f"Successfully added columns: {', '.join(columns_added)}") - logging.info("Updated P.1 station with sample geolocation data") - else: - logging.info("All geolocation columns already exist") - - conn.close() - return True - - except ImportError: - logging.error("pymysql not installed. Run: pip install pymysql") - return False - except Exception as e: - logging.error(f"Error migrating MySQL database: {e}") - return False - -def load_config_from_env() -> Dict[str, Any]: - """Load database configuration from environment variables""" - db_type = os.getenv('DB_TYPE', 'sqlite').lower() - - if db_type == 'postgresql': - return { - 'type': 'postgresql', - 'connection_string': os.getenv('POSTGRES_CONNECTION_STRING', - 'postgresql://postgres:password@localhost/water_monitoring') - } - elif db_type == 'mysql': - return { - 'type': 'mysql', - 'connection_string': os.getenv('MYSQL_CONNECTION_STRING', - 'mysql://root:password@localhost/water_monitoring') - } - elif db_type == 'victoriametrics': - logging.info("VictoriaMetrics doesn't require schema migration") - return {'type': 'victoriametrics'} - elif db_type == 'influxdb': - logging.info("InfluxDB doesn't require schema migration") - return {'type': 'influxdb'} - else: - # Default to SQLite - return { - 'type': 'sqlite', - 'db_path': os.getenv('SQLITE_DB_PATH', 'water_monitoring.db') - } - -def main(): - """Main migration function""" - logging.info("Starting geolocation column migration...") - - # Load configuration - config = load_config_from_env() - db_type = config['type'] - - logging.info(f"Detected database type: {db_type.upper()}") - - success = False - - if db_type == 'sqlite': - db_path = config.get('db_path', 'water_monitoring.db') - if not os.path.exists(db_path): - logging.error(f"Database file not found: {db_path}") - sys.exit(1) - success = migrate_sqlite(db_path) - - elif db_type == 'postgresql': - success = migrate_postgresql(config['connection_string']) - - elif db_type == 'mysql': - success = migrate_mysql(config['connection_string']) - - elif db_type in ['victoriametrics', 'influxdb']: - logging.info(f"{db_type.upper()} doesn't require schema migration") - success = True - - else: - logging.error(f"Unsupported database type: {db_type}") - sys.exit(1) - - if success: - logging.info("✅ Migration completed successfully!") - logging.info("You can now restart your water monitoring application") - logging.info("The system will automatically use the new geolocation columns") - else: - logging.error("❌ Migration failed!") - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/setup.py.backup b/setup.py.backup deleted file mode 100644 index 5d55743..0000000 --- a/setup.py.backup +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 -""" -Setup script for Northern Thailand Ping River Monitor -""" - -from setuptools import setup, find_packages -import os - -# Read the README file -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -# Read requirements -try: - with open("requirements.txt", "r", encoding="utf-8") as fh: - requirements = [line.strip() for line in fh if line.strip() and not line.startswith("#")] -except FileNotFoundError: - # Fallback to minimal requirements if file not found - requirements = [ - "requests>=2.31.0", - "schedule>=1.2.0", - "pandas>=2.1.0", - "fastapi>=0.104.0", - "uvicorn>=0.24.0", - ] - -# Extract core requirements (exclude dev dependencies) -core_requirements = [] -for req in requirements: - if not any(dev_keyword in req.lower() for dev_keyword in ['pytest', 'black', 'flake8', 'mypy', 'sphinx']): - core_requirements.append(req) - -setup( - name="northern-thailand-ping-river-monitor", - version="3.1.3", - author="Ping River Monitor Team", - author_email="contact@example.com", - description="Real-time water level monitoring system for the Ping River Basin in Northern Thailand", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor", - project_urls={ - "Bug Tracker": "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/issues", - "Documentation": "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor/wiki", - "Source Code": "https://git.b4l.co.th/B4L/Northern-Thailand-Ping-River-Monitor", - }, - packages=find_packages(), - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Science/Research", - "Intended Audience :: System Administrators", - "Topic :: Scientific/Engineering :: Hydrology", - "Topic :: System :: Monitoring", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Operating System :: OS Independent", - "Environment :: Web Environment", - "Framework :: FastAPI", - ], - python_requires=">=3.9", - install_requires=core_requirements, - extras_require={ - "dev": [ - "pytest>=7.4.3", - "pytest-cov>=4.1.0", - "black>=23.11.0", - "flake8>=6.1.0", - "mypy>=1.7.1", - "pre-commit>=3.5.0", - ], - "docs": [ - "sphinx>=7.2.6", - "sphinx-rtd-theme>=1.3.0", - ], - "all": [ - "influxdb>=5.3.1", - "pymysql>=1.1.0", - "psycopg2-binary>=2.9.9", - ], - }, - entry_points={ - "console_scripts": [ - "ping-river-monitor=src.main:main", - "ping-river-api=src.web_api:main", - ], - }, - include_package_data=True, - package_data={ - "src": ["*.py"], - }, - keywords=[ - "water monitoring", - "hydrology", - "thailand", - "ping river", - "environmental monitoring", - "time series", - "fastapi", - "real-time data", - ], - zip_safe=False, -) \ No newline at end of file