483 lines
17 KiB
Python
483 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# ========= Imports & chargement .env =========
|
|
import os
|
|
import re
|
|
import time
|
|
import ssl
|
|
import smtplib
|
|
import logging
|
|
import datetime as dt
|
|
from email.message import EmailMessage
|
|
from typing import List
|
|
|
|
from dotenv import load_dotenv, find_dotenv
|
|
load_dotenv(find_dotenv(usecwd=True), override=False)
|
|
SITE = "Saclay"
|
|
PROGRAM_NAME = f"Monitor_{SITE}"
|
|
|
|
# MySQL
|
|
import mysql.connector
|
|
from mysql.connector import Error as MySQLError
|
|
|
|
# OVH (robuste même si la lib n'est pas installée)
|
|
try:
|
|
import ovh
|
|
from ovh.exceptions import APIError as OVHAPIError
|
|
_ovh_available = True
|
|
except Exception:
|
|
ovh = None # type: ignore
|
|
class OVHAPIError(Exception): # fallback pour les except
|
|
pass
|
|
_ovh_available = False
|
|
|
|
# ========= Logger =========
|
|
log = logging.getLogger("monitor_saclay")
|
|
if not log.handlers:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
|
|
# ========= Utilitaires DB =========
|
|
def get_db():
|
|
cnx = mysql.connector.connect(
|
|
host=os.getenv("DB_HOST", "localhost"),
|
|
user=os.getenv("DB_USER"),
|
|
password=os.getenv("DB_PASS"),
|
|
database=os.getenv("DB_NAME", "Sondes"),
|
|
port=int(os.getenv("DB_PORT", "3306")),
|
|
autocommit=True,
|
|
)
|
|
return cnx
|
|
|
|
def lire_sondes_depuis_db(site: str):
|
|
"""
|
|
Relevés les + récents par sonde pour le site (table = nom du site, ex. 'Saclay').
|
|
Retour: [{"Sonde": str, "Temperature": float, "Date": datetime}]
|
|
"""
|
|
table = site
|
|
sql = f"""
|
|
SELECT t1.Sonde, t1.Temperature, t1.Date
|
|
FROM `{table}` t1
|
|
JOIN (
|
|
SELECT Sonde, MAX(Date) AS MaxDate
|
|
FROM `{table}`
|
|
GROUP BY Sonde
|
|
) t2 ON t1.Sonde=t2.Sonde AND t1.Date=t2.MaxDate
|
|
"""
|
|
cnx = get_db()
|
|
try:
|
|
cur = cnx.cursor(dictionary=True)
|
|
cur.execute(sql)
|
|
rows = cur.fetchall()
|
|
for r in rows:
|
|
r["Temperature"] = float(r["Temperature"])
|
|
return rows
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (lire_sondes_depuis_db): %s", err)
|
|
return []
|
|
finally:
|
|
cnx.close()
|
|
|
|
def lire_seuils_depuis_db(site: str):
|
|
"""
|
|
Lit 'Chambres_froides' pour le site (Etat='ON').
|
|
Retour: dict {sonde: seuil_float}
|
|
"""
|
|
sql = """
|
|
SELECT Sonde, Temp_Max
|
|
FROM Chambres_froides
|
|
WHERE Lieu=%s AND Etat='ON'
|
|
"""
|
|
cnx = get_db()
|
|
seuils = {}
|
|
try:
|
|
cur = cnx.cursor()
|
|
cur.execute(sql, (site,))
|
|
for sonde, s in cur.fetchall():
|
|
seuils[str(sonde)] = float(s)
|
|
return seuils
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (lire_seuils_depuis_db): %s", err)
|
|
return seuils
|
|
finally:
|
|
cnx.close()
|
|
|
|
def depassement_depuis_30min(site: str, sonde: str, seuil: float) -> bool:
|
|
"""
|
|
Vrai si dépassement continu >= 30 minutes.
|
|
Approche : cherche la première mesure > seuil dans les 120 dernières minutes,
|
|
vérifie que la dernière mesure est toujours > seuil et que l'écart >= 30 min.
|
|
"""
|
|
table = site
|
|
cnx = get_db()
|
|
try:
|
|
cur = cnx.cursor()
|
|
|
|
# Dernière valeur
|
|
cur.execute(f"""
|
|
SELECT Temperature, Date
|
|
FROM `{table}`
|
|
WHERE Sonde=%s
|
|
ORDER BY Date DESC
|
|
LIMIT 1
|
|
""", (sonde,))
|
|
last = cur.fetchone()
|
|
if not last:
|
|
return False
|
|
last_temp, last_date = float(last[0]), last[1]
|
|
if last_temp <= float(seuil):
|
|
return False
|
|
|
|
# Première mesure > seuil (fenêtre 120 min)
|
|
cur.execute(f"""
|
|
SELECT MIN(Date)
|
|
FROM `{table}`
|
|
WHERE Sonde=%s
|
|
AND Temperature > %s
|
|
AND Date >= (NOW() - INTERVAL 120 MINUTE)
|
|
""", (sonde, float(seuil)))
|
|
first_over = cur.fetchone()[0]
|
|
if not first_over:
|
|
return False
|
|
|
|
now = dt.datetime.now(tz=getattr(first_over, "tzinfo", None))
|
|
return (now - first_over) >= dt.timedelta(minutes=30)
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (depassement_depuis_30min): %s", err)
|
|
return False
|
|
finally:
|
|
cnx.close()
|
|
|
|
def alerte_en_cours(site: str, sonde: str) -> bool:
|
|
table = f"Alertes_{site}"
|
|
cnx = get_db()
|
|
try:
|
|
cur = cnx.cursor()
|
|
cur.execute(
|
|
f"SELECT 1 FROM `{table}` WHERE `Sonde`=%s AND `Etat`='En cours' LIMIT 1",
|
|
(sonde,)
|
|
)
|
|
return cur.fetchone() is not None
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (alerte_en_cours): %s", err)
|
|
return False
|
|
finally:
|
|
cnx.close()
|
|
|
|
def creer_alerte(site: str, sonde: str):
|
|
table = f"Alertes_{site}"
|
|
cnx = get_db()
|
|
try:
|
|
cur = cnx.cursor()
|
|
cur.execute(
|
|
f"INSERT INTO `{table}` (`Sonde`, `Debut_defaut`, `Etat`) VALUES (%s, NOW(), 'En cours')",
|
|
(sonde,)
|
|
)
|
|
cnx.commit()
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (creer_alerte): %s", err)
|
|
finally:
|
|
cnx.close()
|
|
|
|
def acquitter_alerte(site: str, sonde: str):
|
|
table = f"Alertes_{site}"
|
|
cnx = get_db()
|
|
try:
|
|
cur = cnx.cursor()
|
|
cur.execute(
|
|
f"UPDATE `{table}` SET `Etat`='Acquitté' WHERE `Sonde`=%s AND `Etat`='En cours'",
|
|
(sonde,)
|
|
)
|
|
cnx.commit()
|
|
except MySQLError as err:
|
|
log.exception("Erreur DB (acquitter_alerte): %s", err)
|
|
finally:
|
|
cnx.close()
|
|
|
|
# ========= Helpers destinataires =========
|
|
def _split_list(raw: str | None) -> list[str]:
|
|
"""Pour les emails (MAIL_TO) — accepte virgule ou point-virgule."""
|
|
return [x.strip() for x in re.split(r"[;,]", raw or "") if x.strip()]
|
|
|
|
def _parse_labeled_phones(raw: str | None) -> list[tuple[str, str]]:
|
|
"""
|
|
Transforme 'Nom:+336..., Autre:+336...' en [('Nom','+336...'), ('Autre','+336...')]
|
|
Si pas de nom fourni -> ('', '+336...')
|
|
"""
|
|
out: list[tuple[str, str]] = []
|
|
for tok in re.split(r"[;,]", raw or ""):
|
|
tok = tok.strip()
|
|
if not tok:
|
|
continue
|
|
if ":" in tok:
|
|
name, num = tok.split(":", 1)
|
|
out.append((name.strip(), num.strip()))
|
|
else:
|
|
out.append(("", tok))
|
|
return out
|
|
|
|
def _resolve_sms_receivers(labeled: list[tuple[str, str]]) -> list[str]:
|
|
"""
|
|
Applique éventuellement ALERT_SMS_ONLY=Nom1,Nom2 ou numéros directs.
|
|
Si ALERT_SMS_ONLY absent -> tous les numéros.
|
|
"""
|
|
only = os.getenv("ALERT_SMS_ONLY")
|
|
if not only:
|
|
return [num for (_name, num) in labeled]
|
|
allow = {x.strip() for x in re.split(r"[;,]", only) if x.strip()}
|
|
return [num for (name, num) in labeled if (name and name in allow) or (num in allow)]
|
|
|
|
def _human_labeled_list(labeled: list[tuple[str, str]]) -> str:
|
|
"""Michel(+336...), Christian(+336...) pour les logs."""
|
|
return ", ".join([f"{n}({p})" if n else p for n, p in labeled])
|
|
|
|
# ========= Notifications (OVH + SMTP) =========
|
|
class Notifier:
|
|
def __init__(self):
|
|
# OVH
|
|
self.ovh_enabled = _ovh_available and all(
|
|
os.getenv(k) for k in (
|
|
"OVH_APPLICATION_KEY", "OVH_APPLICATION_SECRET", "OVH_CONSUMER_KEY",
|
|
"OVH_SMS_SERVICE", "OVH_SMS_SENDER"
|
|
)
|
|
)
|
|
if self.ovh_enabled:
|
|
self.ovh_client = ovh.Client(
|
|
endpoint=os.getenv("OVH_ENDPOINT", "ovh-eu"),
|
|
application_key=os.getenv("OVH_APPLICATION_KEY"),
|
|
application_secret=os.getenv("OVH_APPLICATION_SECRET"),
|
|
consumer_key=os.getenv("OVH_CONSUMER_KEY"),
|
|
)
|
|
self.ovh_service = os.getenv("OVH_SMS_SERVICE")
|
|
self.ovh_sender = os.getenv("OVH_SMS_SENDER")
|
|
|
|
# <<< LIGNE CLÉ POUR SACLAY >>>
|
|
raw_sms = (os.getenv("ALERT_SMS_TO_Saclay")
|
|
or os.getenv("ALERT_SMS_TO_SACLAY")
|
|
or os.getenv("ALERT_SMS_TO")) # fallback facultatif
|
|
self.sms_labeled = _parse_labeled_phones(raw_sms)
|
|
else:
|
|
self.sms_labeled = []
|
|
|
|
# SMTP
|
|
self.smtp_host = os.getenv("SMTP_HOST")
|
|
self.smtp_port = int(os.getenv("SMTP_PORT", "465"))
|
|
self.smtp_user = os.getenv("SMTP_USER")
|
|
self.smtp_pass = os.getenv("SMTP_PASS")
|
|
self.mail_from = os.getenv("MAIL_FROM") or self.smtp_user
|
|
self.mail_to = _split_list(os.getenv("MAIL_TO") or os.getenv("EMAIL_DESTINATAIRES"))
|
|
self.smtp_security = (os.getenv("SMTP_SECURITY", "SSL") or "SSL").upper()
|
|
# >>> NOUVEAU : destinataires/expéditeur par site, sinon valeur générique
|
|
site_key = SITE # "Saclay" ou "Meudon" selon le fichier
|
|
raw_mail_to = (os.getenv(f"MAIL_TO_{site_key}")
|
|
or os.getenv(f"MAIL_TO_{site_key.upper()}")
|
|
or os.getenv("MAIL_TO")
|
|
or "")
|
|
self.mail_to = _split_list(raw_mail_to)
|
|
|
|
self.mail_from = (os.getenv(f"MAIL_FROM_{site_key}")
|
|
or os.getenv(f"MAIL_FROM_{site_key.upper()}")
|
|
or os.getenv("MAIL_FROM")
|
|
or self.smtp_user)
|
|
|
|
self.smtp_enabled = all([self.smtp_host, self.smtp_port, self.smtp_user, self.smtp_pass, self.mail_to])
|
|
|
|
def send_sms(self, message: str, tag: str = "monitor-saclay") -> bool:
|
|
if not self.ovh_enabled or not self.sms_labeled:
|
|
log.warning("SMS désactivé ou aucun destinataire.")
|
|
return False
|
|
|
|
receivers = _resolve_sms_receivers(self.sms_labeled) # liste de numéros
|
|
if not receivers:
|
|
log.warning("ALERT_SMS_ONLY a filtré tous les destinataires (aucun envoi).")
|
|
return False
|
|
|
|
payload = {
|
|
"sender": self.ovh_sender,
|
|
"receivers": receivers,
|
|
"message": message[:1600],
|
|
"priority": "high",
|
|
"coding": "7bit",
|
|
"class": "phoneDisplay",
|
|
"noStopClause": True, # transactionnel (H24) si habilité chez OVH
|
|
"senderForResponse": False,
|
|
"validityPeriod": 2880,
|
|
"tag": tag,
|
|
}
|
|
|
|
try:
|
|
log.info("Envoi SMS vers: %s", _human_labeled_list([(n, p) for (n, p) in self.sms_labeled if p in receivers]))
|
|
resp = self.ovh_client.post(f"/sms/{self.ovh_service}/jobs", **payload)
|
|
ids = resp.get("ids") or []
|
|
log.info("SMS OVH envoyé (job ids=%s)", ids)
|
|
# Suivi non bloquant : OVH peut supprimer le job très vite → ignorer 404
|
|
try:
|
|
if ids:
|
|
job_id = ids[0]
|
|
for _ in range(3):
|
|
job = self.ovh_client.get(f"/sms/{self.ovh_service}/jobs/{job_id}")
|
|
if job.get("status") in ("done", "error", "cancelled"):
|
|
log.info("Statut job SMS: %s", job.get("status"))
|
|
break
|
|
time.sleep(1.5)
|
|
except Exception as err:
|
|
log.debug("Suivi job OVH indisponible (OK): %s", err)
|
|
return True
|
|
|
|
except OVHAPIError as err:
|
|
log.exception("Erreur API OVH: %s", err)
|
|
return False
|
|
except Exception as err:
|
|
log.exception("Echec envoi SMS OVH: %s", err)
|
|
return False
|
|
|
|
def send_email(self, subject: str, body: str) -> bool:
|
|
if not self.smtp_enabled:
|
|
log.warning("SMTP non configuré, email non envoyé.")
|
|
return False
|
|
|
|
msg = EmailMessage()
|
|
msg["From"] = self.mail_from
|
|
msg["To"] = ", ".join(self.mail_to)
|
|
msg["Subject"] = subject
|
|
msg.set_content(body)
|
|
|
|
try:
|
|
if self.smtp_security == "STARTTLS":
|
|
with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=20) as server:
|
|
server.ehlo()
|
|
server.starttls(context=ssl.create_default_context())
|
|
server.ehlo()
|
|
server.login(self.smtp_user, self.smtp_pass)
|
|
server.send_message(msg)
|
|
else:
|
|
with smtplib.SMTP_SSL(self.smtp_host, self.smtp_port, context=ssl.create_default_context(), timeout=20) as server:
|
|
server.login(self.smtp_user, self.smtp_pass)
|
|
server.send_message(msg)
|
|
log.info("Email envoyé à %s", self.mail_to)
|
|
return True
|
|
except (smtplib.SMTPException, ssl.SSLError) as err:
|
|
log.exception("Erreur SMTP: %s", err)
|
|
return False
|
|
except Exception as err:
|
|
log.exception("Echec envoi email: %s", err)
|
|
return False
|
|
|
|
# ========= Helpers de mise en forme des messages =========
|
|
from zoneinfo import ZoneInfo
|
|
PARIS = ZoneInfo("Europe/Paris")
|
|
|
|
def fmt_deg(v: float) -> str:
|
|
s = f"{float(v):.1f}".replace(".", ",")
|
|
return f"{s}°C"
|
|
|
|
def now_paris() -> dt.datetime:
|
|
return dt.datetime.now(tz=PARIS)
|
|
|
|
def build_alert_text(site: str, sonde: str, temp: float, seuil: float, when: dt.datetime | None = None):
|
|
when = when or now_paris()
|
|
subject = f"[ALERTE {site}] {sonde} au-dessus du seuil"
|
|
lines = [
|
|
subject + ":",
|
|
f"Sonde: {sonde}",
|
|
f"Température: {fmt_deg(temp)} (seuil {fmt_deg(seuil)})",
|
|
f"Site: {site}",
|
|
f"Heure: {when.strftime('%Y-%m-%d %H:%M:%S')}",
|
|
]
|
|
txt = "\n".join(lines)
|
|
return subject, txt, txt
|
|
|
|
def build_ok_text(site: str, sonde: str, temp: float, seuil: float, when: dt.datetime | None = None):
|
|
when = when or now_paris()
|
|
subject = f"[OK {site}] {sonde} revenue normale"
|
|
lines = [
|
|
subject + ":",
|
|
f"Sonde: {sonde}",
|
|
f"Température: {fmt_deg(temp)} <= seuil {fmt_deg(seuil)}",
|
|
f"Site: {site}",
|
|
f"Heure: {when.strftime('%Y-%m-%d %H:%M:%S')}",
|
|
]
|
|
txt = "\n".join(lines)
|
|
return subject, txt, txt
|
|
|
|
# ========= Fonctions de notification haut niveau =========
|
|
notifier = Notifier()
|
|
|
|
def notifier_sur_depassement(site: str, sonde: str, temp: float, seuil: float):
|
|
subject, sms_text, email_body = build_alert_text(site, sonde, temp, seuil)
|
|
sms_ok = notifier.send_sms(sms_text)
|
|
mail_ok = notifier.send_email(subject, email_body)
|
|
if sms_ok and mail_ok:
|
|
log.info("Alerte envoyée (SMS+mail) pour %s/%s", site, sonde)
|
|
elif sms_ok:
|
|
log.warning("Alerte: SMS OK, mail KO pour %s/%s", site, sonde)
|
|
elif mail_ok:
|
|
log.warning("Alerte: mail OK, SMS KO pour %s/%s", site, sonde)
|
|
else:
|
|
log.error("Alerte: SMS et mail KO pour %s/%s", site, sonde)
|
|
|
|
def notifier_acquittement(site: str, sonde: str, temp: float, seuil: float):
|
|
subject, sms_text, _ = build_ok_text(site, sonde, temp, seuil)
|
|
sms_ok = notifier.send_sms(sms_text) # retour à la normale: SMS seul
|
|
if sms_ok:
|
|
log.info("Acquittement envoyé (SMS) pour %s/%s", site, sonde)
|
|
else:
|
|
log.warning("Acquittement: SMS KO pour %s/%s", site, sonde)
|
|
|
|
# ========= Cycle & boucle de monitoring =========
|
|
def run_monitor_cycle(site: str):
|
|
sondes = lire_sondes_depuis_db(site)
|
|
seuils = lire_seuils_depuis_db(site)
|
|
|
|
for r in sondes:
|
|
nom = str(r["Sonde"])
|
|
temp = float(r["Temperature"])
|
|
seuil = float(seuils.get(nom, 6.0)) # défaut 6°C si manquant
|
|
|
|
if temp > seuil:
|
|
if depassement_depuis_30min(site, nom, seuil) and not alerte_en_cours(site, nom):
|
|
creer_alerte(site, nom)
|
|
notifier_sur_depassement(site, nom, temp, seuil)
|
|
else:
|
|
if alerte_en_cours(site, nom):
|
|
acquitter_alerte(site, nom)
|
|
notifier_acquittement(site, nom, temp, seuil)
|
|
|
|
def run_monitor_loop(site: str, period_sec: int = 300):
|
|
log.info("Monitor_Saclay démarré (site=%s, période=%ss) ✅", site, period_sec)
|
|
while True:
|
|
t0 = time.time()
|
|
try:
|
|
run_monitor_cycle(site)
|
|
except Exception as err: # volontaire : ne jamais tuer le service
|
|
log.exception("Erreur cycle monitoring: %s", err)
|
|
time.sleep(max(0, period_sec - (time.time() - t0)))
|
|
|
|
# ========= Entrée CLI =========
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="Monitor_Saclay")
|
|
parser.add_argument("--site", default=os.getenv("SITE_NAME", "Saclay"))
|
|
parser.add_argument("--period", type=int, default=300, help="période en secondes (défaut 300)")
|
|
# tests
|
|
parser.add_argument("--test-sms", action="store_true")
|
|
parser.add_argument("--test-mail", action="store_true")
|
|
parser.add_argument("--test-alert", action="store_true")
|
|
parser.add_argument("--test-ok", action="store_true")
|
|
parser.add_argument("--once", action="store_true", help="exécuter un seul cycle puis quitter")
|
|
args = parser.parse_args()
|
|
|
|
if args.test_sms:
|
|
notifier.send_sms("TEST DOMO91 (transactionnel)")
|
|
elif args.test_mail:
|
|
notifier.send_email("[TEST DOMO91] Mail", "OK")
|
|
elif args.test_alert:
|
|
notifier_sur_depassement(args.site, "Congelateur", -14.5, -15.0)
|
|
elif args.test_ok:
|
|
notifier_acquittement(args.site, "Congelateur", -15.2, -15.0)
|
|
else:
|
|
if args.once:
|
|
run_monitor_cycle(args.site)
|
|
else:
|
|
run_monitor_loop(args.site, period_sec=args.period)
|