- log_buffer: RingBufferHandler, кольцевой буфер 600 записей, fan-out SSE - ui_logs: GET /ui/logs (HTML), GET /ui/logs/stream (EventSource) - main: install_log_handler при старте, подключён router логов - nav_rail: ссылка Логи, root_html: кнопка-ссылка Логи - Исправлено: NaN/Inf/NUL в теле вебхука → 500 от PostgreSQL jsonb - Тесты: test_log_buffer, test_json_sanitize; 51 passed Made-with: Cursor
216 lines
8.1 KiB
Python
216 lines
8.1 KiB
Python
import json
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from urllib.parse import urlparse
|
|
|
|
from fastapi import APIRouter, Depends, Header, HTTPException, Request
|
|
from starlette.responses import Response
|
|
|
|
from onguard24.domain.entities import Alert, Severity
|
|
from onguard24.grafana_sources import sources_by_slug, webhook_authorized
|
|
from onguard24.ingress.grafana_payload import extract_alert_row_from_grafana_body
|
|
from onguard24.ingress.json_sanitize import sanitize_for_jsonb
|
|
from onguard24.ingress.team_match import resolve_team_id_for_labels
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(tags=["ingress"])
|
|
|
|
_PATH_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
|
|
|
|
|
def sanitize_source_key(raw: str) -> str:
|
|
s = re.sub(r"[^a-zA-Z0-9._-]+", "-", raw.strip()).strip("-").lower()
|
|
return s[:200] if s else ""
|
|
|
|
|
|
def extract_grafana_source_key(body: dict) -> str | None:
|
|
"""
|
|
Идентификатор инстанса/организации из тела вебхука Grafana (без ручной прописки в .env).
|
|
См. поля payload: externalURL, orgId, лейблы в alerts[].
|
|
"""
|
|
chunks: list[str] = []
|
|
ext = body.get("externalURL") or body.get("external_url")
|
|
if isinstance(ext, str) and ext.strip():
|
|
try:
|
|
host = urlparse(ext.strip()).hostname
|
|
if host:
|
|
chunks.append(host)
|
|
except Exception:
|
|
pass
|
|
for key in ("orgId", "org_id"):
|
|
v = body.get(key)
|
|
if v is not None and str(v).strip():
|
|
chunks.append(f"o{str(v).strip()[:24]}")
|
|
break
|
|
alerts = body.get("alerts")
|
|
if isinstance(alerts, list) and alerts:
|
|
a0 = alerts[0] if isinstance(alerts[0], dict) else {}
|
|
labels = a0.get("labels") if isinstance(a0.get("labels"), dict) else {}
|
|
for lk in ("__org_id__", "grafana_org", "tenant", "cluster", "namespace"):
|
|
v = labels.get(lk)
|
|
if v is not None and str(v).strip():
|
|
chunks.append(str(v).strip()[:64])
|
|
break
|
|
if not chunks:
|
|
return None
|
|
return sanitize_source_key("-".join(chunks)) or None
|
|
|
|
|
|
async def get_pool(request: Request):
|
|
return getattr(request.app.state, "pool", None)
|
|
|
|
|
|
def service_hint_from_grafana_body(body: dict, header_service: str | None) -> str | None:
|
|
"""Имя сервиса: заголовок X-OnGuard-Service или лейблы из Unified Alerting."""
|
|
if header_service and header_service.strip():
|
|
return header_service.strip()[:200]
|
|
alerts = body.get("alerts")
|
|
if isinstance(alerts, list) and alerts:
|
|
labels = alerts[0].get("labels") if isinstance(alerts[0], dict) else None
|
|
if isinstance(labels, dict):
|
|
for key in ("service", "job", "namespace", "cluster", "instance"):
|
|
v = labels.get(key)
|
|
if v is not None and str(v).strip():
|
|
return str(v).strip()[:200]
|
|
common = body.get("commonLabels")
|
|
if isinstance(common, dict):
|
|
for key in ("service", "job", "namespace"):
|
|
v = common.get(key)
|
|
if v is not None and str(v).strip():
|
|
return str(v).strip()[:200]
|
|
return None
|
|
|
|
|
|
async def _grafana_webhook_impl(
|
|
request: Request,
|
|
pool,
|
|
x_onguard_secret: str | None,
|
|
x_onguard_service: str | None,
|
|
path_slug: str | None,
|
|
) -> Response:
|
|
settings = request.app.state.settings
|
|
|
|
raw = await request.body()
|
|
if len(raw) > 1_000_000:
|
|
raise HTTPException(status_code=400, detail="body too large")
|
|
try:
|
|
body = json.loads(raw.decode() or "{}")
|
|
except json.JSONDecodeError:
|
|
body = {}
|
|
if not isinstance(body, dict):
|
|
body = {}
|
|
else:
|
|
body = sanitize_for_jsonb(body)
|
|
|
|
derived = extract_grafana_source_key(body)
|
|
path_key: str | None = None
|
|
if path_slug is not None:
|
|
path_key = path_slug.strip().lower()
|
|
if not _PATH_SLUG_RE.match(path_key):
|
|
raise HTTPException(status_code=400, detail="invalid path slug")
|
|
stored_org_slug = path_key
|
|
else:
|
|
stored_org_slug = derived
|
|
|
|
by = sources_by_slug(settings)
|
|
source = by.get(path_key) if path_key else None
|
|
if not webhook_authorized(settings, source, x_onguard_secret):
|
|
raise HTTPException(status_code=401, detail="unauthorized")
|
|
|
|
service_name = service_hint_from_grafana_body(body, x_onguard_service)
|
|
|
|
if pool is None:
|
|
logger.warning("ingress: database not configured, event not persisted")
|
|
return Response(status_code=202)
|
|
|
|
title_row, sev_row, labels_row, fp_row = extract_alert_row_from_grafana_body(body)
|
|
async with pool.acquire() as conn:
|
|
async with conn.transaction():
|
|
row = await conn.fetchrow(
|
|
"""
|
|
INSERT INTO ingress_events (source, body, org_slug, service_name)
|
|
VALUES ($1, $2::jsonb, $3, $4)
|
|
RETURNING id
|
|
""",
|
|
"grafana",
|
|
json.dumps(body, ensure_ascii=False, allow_nan=False),
|
|
stored_org_slug,
|
|
service_name,
|
|
)
|
|
raw_id = row["id"] if row else None
|
|
if raw_id is not None:
|
|
team_id = await resolve_team_id_for_labels(conn, labels_row)
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO irm_alerts (
|
|
ingress_event_id, status, title, severity, source,
|
|
grafana_org_slug, service_name, labels, fingerprint, team_id
|
|
)
|
|
VALUES ($1, 'firing', $2, $3, 'grafana', $4, $5, $6::jsonb, $7, $8)
|
|
""",
|
|
raw_id,
|
|
title_row or "—",
|
|
sev_row,
|
|
stored_org_slug,
|
|
service_name,
|
|
json.dumps(labels_row, ensure_ascii=False, allow_nan=False),
|
|
fp_row,
|
|
team_id,
|
|
)
|
|
bus = getattr(request.app.state, "event_bus", None)
|
|
if bus and raw_id is not None:
|
|
title = str(body.get("title") or body.get("ruleName") or "")[:500]
|
|
alert = Alert(
|
|
source="grafana",
|
|
title=title,
|
|
severity=Severity.WARNING,
|
|
payload=body,
|
|
received_at=datetime.now(timezone.utc),
|
|
)
|
|
try:
|
|
await bus.publish_alert_received(
|
|
alert,
|
|
raw_payload_ref=raw_id,
|
|
grafana_org_slug=stored_org_slug,
|
|
service_name=service_name,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"ingress: событие alert.received не доставлено подписчикам (БД уже сохранена)"
|
|
)
|
|
return Response(status_code=202)
|
|
|
|
|
|
@router.post("/ingress/grafana", status_code=202)
|
|
async def grafana_webhook_legacy(
|
|
request: Request,
|
|
pool=Depends(get_pool),
|
|
x_onguard_secret: str | None = Header(default=None, alias="X-OnGuard-Secret"),
|
|
x_onguard_service: str | None = Header(default=None, alias="X-OnGuard-Service"),
|
|
):
|
|
"""
|
|
Универсальный URL для любого инстанса Grafana: org_slug в БД берётся из тела
|
|
(externalURL, orgId, лейблы), без преднастройки в .env.
|
|
"""
|
|
return await _grafana_webhook_impl(
|
|
request, pool, x_onguard_secret, x_onguard_service, path_slug=None
|
|
)
|
|
|
|
|
|
@router.post("/ingress/grafana/{org_slug}", status_code=202)
|
|
async def grafana_webhook_org(
|
|
org_slug: str,
|
|
request: Request,
|
|
pool=Depends(get_pool),
|
|
x_onguard_secret: str | None = Header(default=None, alias="X-OnGuard-Secret"),
|
|
x_onguard_service: str | None = Header(default=None, alias="X-OnGuard-Service"),
|
|
):
|
|
"""
|
|
Опционально: явный ярлык в URL (перекрывает авто-извлечение из JSON).
|
|
Секрет для пути: webhook_secret из GRAFANA_SOURCES_JSON для этого slug, иначе общий.
|
|
"""
|
|
return await _grafana_webhook_impl(
|
|
request, pool, x_onguard_secret, x_onguard_service, path_slug=org_slug
|
|
)
|