Mei_Sheng_Textiles/POE-sensor
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor configuration and health check server in POE project. Updated config.py to use environment variables for MQTT and health check settings, enhancing flexibility. Improved health check server in health_check.py with readiness endpoint and better error handling. Main application in main.py now includes graceful shutdown handling and health server verification. Adjusted Nomad job configuration for improved resource allocation and health check parameters.

Browse Source
This commit is contained in:
Naab2k3
2025-06-23 13:42:58 +07:00
parent c515b6d2e9
commit 8991a02bf5
4 changed files with 249 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
health_check.py
View File

@ -4,6 +4,8 @@ from http.server import HTTPServer, BaseHTTPRequestHandler
from datetime import datetime, timezone
import logging
import time
import socket
import os
from config import HEALTH_CHECK_PORT, HEALTH_CHECK_ENABLED
@ -13,6 +15,8 @@ class HealthCheckHandler(BaseHTTPRequestHandler):
self.send_health_response()
elif self.path == '/sensors':
self.send_sensors_status()
elif self.path == '/ready':
self.send_readiness_response()
else:
self.send_response(404)
self.end_headers()
@ -33,16 +37,24 @@ class HealthCheckHandler(BaseHTTPRequestHandler):
"sensors": {
"total": summary.get('total_sensors', 0),
"online": summary.get('online_sensors', 0),
"offline": summary.get('offline_sensors', 0),
"unknown": summary.get('unknown_sensors', 0),
"health_percentage": summary.get('health_percentage', 0.0)
}
},
"uptime": self._get_uptime(),
"host": socket.gethostname()
}
# Consider the service unhealthy if no sensors are working
if summary.get('total_sensors', 0) > 0 and summary.get('online_sensors', 0) == 0:
# If we have sensors configured but none are online, report as degraded but still healthy
# (since the health check server itself is working)
health_data["status"] = "degraded"
health_data["message"] = "All sensors offline"
# Service is healthy if the health check server is running
# Don't mark as unhealthy just because sensors are offline
# That's what the degraded status is for
if summary.get('total_sensors', 0) > 0:
if summary.get('online_sensors', 0) == 0:
health_data["status"] = "degraded"
health_data["message"] = "All sensors offline but service is running"
elif summary.get('health_percentage', 0) < 50:
health_data["status"] = "degraded"
health_data["message"] = f"Low sensor health: {summary.get('health_percentage', 0):.1f}%"
except Exception as e:
# If we can't get sensor status, still report as healthy since the service is running
@ -52,44 +64,115 @@ class HealthCheckHandler(BaseHTTPRequestHandler):
"timestamp": datetime.now(timezone.utc).isoformat(),
"service": "modbus-mqtt-bridge",
"version": "1.0.0",
"message": "Service running, sensor status unavailable"
"message": "Service running, sensor status unavailable",
"uptime": self._get_uptime(),
"host": socket.gethostname()
}
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(json.dumps(health_data, indent=2).encode())
def send_readiness_response(self):
"""Send readiness probe response - checks if service is ready to serve"""
try:
# Check if main components are initialized
from sensor_tracker import get_sensor_tracker
sensor_tracker = get_sensor_tracker()
ready_data = {
"ready": True,
"timestamp": datetime.now(timezone.utc).isoformat(),
"service": "modbus-mqtt-bridge",
"checks": {
"sensor_tracker": True,
"health_server": True
}
}
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(json.dumps(ready_data, indent=2).encode())
except Exception as e:
logging.error(f"Readiness check failed: {e}")
ready_data = {
"ready": False,
"timestamp": datetime.now(timezone.utc).isoformat(),
"service": "modbus-mqtt-bridge",
"error": str(e)
}
self.send_response(503)
self.send_header('Content-type', 'application/json')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(json.dumps(ready_data, indent=2).encode())
def send_sensors_status(self):
"""Send detailed sensor status"""
# Get sensor status from the global sensor tracker
from sensor_tracker import get_all_sensor_status
sensors_status = get_all_sensor_status()
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps(sensors_status, indent=2).encode())
try:
# Get sensor status from the global sensor tracker
from sensor_tracker import get_all_sensor_status
sensors_status = get_all_sensor_status()
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Cache-Control', 'no-cache')
self.end_headers()
self.wfile.write(json.dumps(sensors_status, indent=2).encode())
except Exception as e:
logging.error(f"Error getting sensor status: {e}")
error_response = {
"error": "Failed to get sensor status",
"message": str(e),
"timestamp": datetime.now(timezone.utc).isoformat()
}
self.send_response(500)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps(error_response, indent=2).encode())
def _get_uptime(self):
"""Get service uptime"""
try:
with open('/proc/uptime', 'r') as f:
uptime_seconds = float(f.readline().split()[0])
return f"{uptime_seconds:.1f}s"
except:
return "unknown"
def log_message(self, format, *args):
"""Override to use our logging system"""
logging.info(f"Health Check - {format % args}")
class HealthCheckServer:
def __init__(self, port=HEALTH_CHECK_PORT):
self.port = port
def __init__(self, port=None):
self.port = port or int(os.getenv('HEALTH_CHECK_PORT', HEALTH_CHECK_PORT))
self.server = None
self.thread = None
self.started = False
def start(self):
"""Start the health check server in a separate thread"""
if not HEALTH_CHECK_ENABLED:
logging.info("Health check server is disabled")
return
return False
try:
logging.info(f"Attempting to start health check server on port {self.port}")
# Check if port is available
if not self._is_port_available(self.port):
logging.error(f"Port {self.port} is already in use. Cannot start health check server.")
return False
# Bind to all interfaces to make it accessible from outside container
self.server = HTTPServer(('0.0.0.0', self.port), HealthCheckHandler)
@ -99,24 +182,50 @@ class HealthCheckServer:
self.thread = threading.Thread(target=self._serve_with_error_handling, daemon=True)
self.thread.start()
# Give the server a moment to start
time.sleep(0.5)
# Give the server a moment to start and verify it's working
time.sleep(1)
logging.info(f"Health check server started on 0.0.0.0:{self.port}")
logging.info(f"Health check endpoints:")
logging.info(f" - http://0.0.0.0:{self.port}/health")
logging.info(f" - http://0.0.0.0:{self.port}/sensors")
logging.info("Health check server is ready for external health checks")
if self._test_health_endpoint():
self.started = True
logging.info(f"Health check server started successfully on 0.0.0.0:{self.port}")
logging.info(f"Health check endpoints:")
logging.info(f" - http://0.0.0.0:{self.port}/health")
logging.info(f" - http://0.0.0.0:{self.port}/sensors")
logging.info(f" - http://0.0.0.0:{self.port}/ready")
logging.info("Health check server is ready for external health checks")
return True
else:
logging.error("Health check server started but endpoints are not responding")
return False
except OSError as e:
if e.errno == 98: # Address already in use
logging.error(f"Port {self.port} is already in use. Cannot start health check server.")
else:
logging.error(f"OS error starting health check server: {e}")
raise e
return False
except Exception as e:
logging.error(f"Failed to start health check server: {e}")
raise e # Re-raise to make the issue visible
return False
def _is_port_available(self, port):
"""Check if port is available"""
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(('0.0.0.0', port))
return True
except OSError:
return False
def _test_health_endpoint(self):
"""Test if health endpoint is responding"""
try:
import urllib.request
with urllib.request.urlopen(f'http://localhost:{self.port}/health', timeout=5) as response:
return response.status == 200
except Exception as e:
logging.warning(f"Health endpoint test failed: {e}")
return False
def _serve_with_error_handling(self):
"""Serve forever with error handling"""
@ -128,10 +237,16 @@ class HealthCheckServer:
logging.error("Health check server is None, cannot serve requests")
except Exception as e:
logging.error(f"Health check server error: {e}", exc_info=True)
self.started = False
def stop(self):
"""Stop the health check server"""
if self.server:
self.server.shutdown()
self.server.server_close()
self.started = False
logging.info("Health check server stopped")
def is_running(self):
"""Check if server is running"""
return self.started and self.thread and self.thread.is_alive()