2026-04-03 09:03:16 +03:00
|
|
|
|
"""IRM: инциденты — учёт сбоев, связь с сырым ingress и событием alert.received."""
|
|
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
|
|
import html
|
|
|
|
|
|
import logging
|
|
|
|
|
|
from uuid import UUID
|
|
|
|
|
|
|
|
|
|
|
|
import asyncpg
|
|
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
|
|
|
|
from fastapi.responses import HTMLResponse
|
|
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
|
|
|
|
|
|
from onguard24.deps import get_pool
|
|
|
|
|
|
from onguard24.domain.events import AlertReceived, DomainEvent, EventBus
|
|
|
|
|
|
from onguard24.modules.ui_support import wrap_module_html_page
|
|
|
|
|
|
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
router = APIRouter(tags=["module-incidents"])
|
|
|
|
|
|
ui_router = APIRouter(tags=["web-incidents"], include_in_schema=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class IncidentCreate(BaseModel):
|
|
|
|
|
|
title: str = Field(..., min_length=1, max_length=500)
|
|
|
|
|
|
status: str = Field(default="open", max_length=64)
|
|
|
|
|
|
severity: str = Field(default="warning", max_length=32)
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-04-03 13:53:19 +03:00
|
|
|
|
class IncidentPatch(BaseModel):
|
|
|
|
|
|
title: str | None = Field(default=None, min_length=1, max_length=500)
|
|
|
|
|
|
status: str | None = Field(default=None, max_length=64)
|
|
|
|
|
|
severity: str | None = Field(default=None, max_length=32)
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-04-03 09:03:16 +03:00
|
|
|
|
def register_events(bus: EventBus, pool: asyncpg.Pool | None = None) -> None:
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
|
|
async def on_alert(ev: DomainEvent) -> None:
|
|
|
|
|
|
if not isinstance(ev, AlertReceived) or ev.raw_payload_ref is None:
|
|
|
|
|
|
return
|
|
|
|
|
|
a = ev.alert
|
|
|
|
|
|
title = (a.title if a else "Алерт без названия")[:500]
|
|
|
|
|
|
sev = (a.severity.value if a else "warning")
|
|
|
|
|
|
try:
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
await conn.execute(
|
|
|
|
|
|
"""
|
2026-04-03 13:53:19 +03:00
|
|
|
|
INSERT INTO incidents (
|
|
|
|
|
|
title, status, severity, source, ingress_event_id,
|
|
|
|
|
|
grafana_org_slug, service_name
|
|
|
|
|
|
)
|
|
|
|
|
|
VALUES ($1, 'open', $2, 'grafana', $3::uuid, $4, $5)
|
2026-04-03 09:03:16 +03:00
|
|
|
|
""",
|
|
|
|
|
|
title,
|
|
|
|
|
|
sev,
|
|
|
|
|
|
ev.raw_payload_ref,
|
2026-04-03 13:53:19 +03:00
|
|
|
|
ev.grafana_org_slug,
|
|
|
|
|
|
ev.service_name,
|
2026-04-03 09:03:16 +03:00
|
|
|
|
)
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
log.exception("incidents: не удалось создать инцидент из alert.received")
|
|
|
|
|
|
|
|
|
|
|
|
bus.subscribe("alert.received", on_alert)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def render_home_fragment(request: Request) -> str:
|
|
|
|
|
|
pool = get_pool(request)
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
return '<p class="module-note">Нужна БД для списка инцидентов.</p>'
|
|
|
|
|
|
try:
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
n = await conn.fetchval("SELECT count(*)::int FROM incidents")
|
|
|
|
|
|
except Exception:
|
|
|
|
|
|
return '<p class="module-note">Таблица инцидентов недоступна (миграции?).</p>'
|
|
|
|
|
|
return f'<div class="module-fragment"><p>Инцидентов в учёте: <strong>{int(n)}</strong></p></div>'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@router.get("/")
|
|
|
|
|
|
async def list_incidents_api(
|
|
|
|
|
|
pool: asyncpg.Pool | None = Depends(get_pool),
|
|
|
|
|
|
limit: int = 50,
|
2026-04-03 13:53:19 +03:00
|
|
|
|
grafana_org_slug: str | None = None,
|
|
|
|
|
|
service_name: str | None = None,
|
2026-04-03 09:03:16 +03:00
|
|
|
|
):
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
return {"items": [], "database": "disabled"}
|
|
|
|
|
|
limit = min(max(limit, 1), 200)
|
2026-04-03 13:53:19 +03:00
|
|
|
|
conditions: list[str] = []
|
|
|
|
|
|
args: list = []
|
|
|
|
|
|
if grafana_org_slug and grafana_org_slug.strip():
|
|
|
|
|
|
args.append(grafana_org_slug.strip())
|
|
|
|
|
|
conditions.append(f"grafana_org_slug = ${len(args)}")
|
|
|
|
|
|
if service_name and service_name.strip():
|
|
|
|
|
|
args.append(service_name.strip())
|
|
|
|
|
|
conditions.append(f"service_name = ${len(args)}")
|
|
|
|
|
|
where_sql = ("WHERE " + " AND ".join(conditions)) if conditions else ""
|
|
|
|
|
|
args.append(limit)
|
|
|
|
|
|
lim_ph = f"${len(args)}"
|
|
|
|
|
|
q = f"""
|
|
|
|
|
|
SELECT id, title, status, severity, source, ingress_event_id, created_at, updated_at,
|
|
|
|
|
|
grafana_org_slug, service_name
|
2026-04-03 09:03:16 +03:00
|
|
|
|
FROM incidents
|
2026-04-03 13:53:19 +03:00
|
|
|
|
{where_sql}
|
2026-04-03 09:03:16 +03:00
|
|
|
|
ORDER BY created_at DESC
|
2026-04-03 13:53:19 +03:00
|
|
|
|
LIMIT {lim_ph}
|
|
|
|
|
|
"""
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
rows = await conn.fetch(q, *args)
|
2026-04-03 09:03:16 +03:00
|
|
|
|
items = []
|
|
|
|
|
|
for r in rows:
|
|
|
|
|
|
items.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
"id": str(r["id"]),
|
|
|
|
|
|
"title": r["title"],
|
|
|
|
|
|
"status": r["status"],
|
|
|
|
|
|
"severity": r["severity"],
|
|
|
|
|
|
"source": r["source"],
|
|
|
|
|
|
"ingress_event_id": str(r["ingress_event_id"]) if r["ingress_event_id"] else None,
|
|
|
|
|
|
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
|
2026-04-03 13:53:19 +03:00
|
|
|
|
"updated_at": r["updated_at"].isoformat() if r.get("updated_at") else None,
|
|
|
|
|
|
"grafana_org_slug": r.get("grafana_org_slug"),
|
|
|
|
|
|
"service_name": r.get("service_name"),
|
2026-04-03 09:03:16 +03:00
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
return {"items": items}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@router.post("/", status_code=201)
|
|
|
|
|
|
async def create_incident_api(
|
|
|
|
|
|
body: IncidentCreate,
|
|
|
|
|
|
pool: asyncpg.Pool | None = Depends(get_pool),
|
|
|
|
|
|
):
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
raise HTTPException(status_code=503, detail="database disabled")
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
row = await conn.fetchrow(
|
|
|
|
|
|
"""
|
2026-04-03 13:53:19 +03:00
|
|
|
|
INSERT INTO incidents (title, status, severity, source, grafana_org_slug, service_name)
|
|
|
|
|
|
VALUES ($1, $2, $3, 'manual', NULL, NULL)
|
|
|
|
|
|
RETURNING id, title, status, severity, source, ingress_event_id, created_at, updated_at,
|
|
|
|
|
|
grafana_org_slug, service_name
|
2026-04-03 09:03:16 +03:00
|
|
|
|
""",
|
|
|
|
|
|
body.title.strip(),
|
|
|
|
|
|
body.status,
|
|
|
|
|
|
body.severity,
|
|
|
|
|
|
)
|
|
|
|
|
|
return {
|
|
|
|
|
|
"id": str(row["id"]),
|
|
|
|
|
|
"title": row["title"],
|
|
|
|
|
|
"status": row["status"],
|
|
|
|
|
|
"severity": row["severity"],
|
|
|
|
|
|
"source": row["source"],
|
|
|
|
|
|
"ingress_event_id": None,
|
|
|
|
|
|
"created_at": row["created_at"].isoformat() if row["created_at"] else None,
|
2026-04-03 13:53:19 +03:00
|
|
|
|
"updated_at": row["updated_at"].isoformat() if row.get("updated_at") else None,
|
|
|
|
|
|
"grafana_org_slug": row.get("grafana_org_slug"),
|
|
|
|
|
|
"service_name": row.get("service_name"),
|
2026-04-03 09:03:16 +03:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-04-03 13:53:19 +03:00
|
|
|
|
def _incident_row_dict(row) -> dict:
|
|
|
|
|
|
return {
|
|
|
|
|
|
"id": str(row["id"]),
|
|
|
|
|
|
"title": row["title"],
|
|
|
|
|
|
"status": row["status"],
|
|
|
|
|
|
"severity": row["severity"],
|
|
|
|
|
|
"source": row["source"],
|
|
|
|
|
|
"ingress_event_id": str(row["ingress_event_id"]) if row["ingress_event_id"] else None,
|
|
|
|
|
|
"created_at": row["created_at"].isoformat() if row["created_at"] else None,
|
|
|
|
|
|
"updated_at": row["updated_at"].isoformat() if row.get("updated_at") else None,
|
|
|
|
|
|
"grafana_org_slug": row.get("grafana_org_slug"),
|
|
|
|
|
|
"service_name": row.get("service_name"),
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@router.get("/{incident_id}/tasks")
|
|
|
|
|
|
async def list_incident_tasks_api(
|
|
|
|
|
|
incident_id: UUID,
|
|
|
|
|
|
pool: asyncpg.Pool | None = Depends(get_pool),
|
|
|
|
|
|
limit: int = 100,
|
|
|
|
|
|
):
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
raise HTTPException(status_code=503, detail="database disabled")
|
|
|
|
|
|
limit = min(max(limit, 1), 200)
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
exists = await conn.fetchval("SELECT 1 FROM incidents WHERE id = $1::uuid", incident_id)
|
|
|
|
|
|
if not exists:
|
|
|
|
|
|
raise HTTPException(status_code=404, detail="incident not found")
|
|
|
|
|
|
rows = await conn.fetch(
|
|
|
|
|
|
"""
|
|
|
|
|
|
SELECT id, incident_id, title, status, created_at
|
|
|
|
|
|
FROM tasks WHERE incident_id = $1::uuid
|
|
|
|
|
|
ORDER BY created_at DESC LIMIT $2
|
|
|
|
|
|
""",
|
|
|
|
|
|
incident_id,
|
|
|
|
|
|
limit,
|
|
|
|
|
|
)
|
|
|
|
|
|
items = []
|
|
|
|
|
|
for r in rows:
|
|
|
|
|
|
items.append(
|
|
|
|
|
|
{
|
|
|
|
|
|
"id": str(r["id"]),
|
|
|
|
|
|
"incident_id": str(r["incident_id"]) if r["incident_id"] else None,
|
|
|
|
|
|
"title": r["title"],
|
|
|
|
|
|
"status": r["status"],
|
|
|
|
|
|
"created_at": r["created_at"].isoformat() if r["created_at"] else None,
|
|
|
|
|
|
}
|
|
|
|
|
|
)
|
|
|
|
|
|
return {"incident_id": str(incident_id), "items": items}
|
|
|
|
|
|
|
|
|
|
|
|
|
2026-04-03 09:03:16 +03:00
|
|
|
|
@router.get("/{incident_id}")
|
|
|
|
|
|
async def get_incident_api(incident_id: UUID, pool: asyncpg.Pool | None = Depends(get_pool)):
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
raise HTTPException(status_code=503, detail="database disabled")
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
row = await conn.fetchrow(
|
|
|
|
|
|
"""
|
2026-04-03 13:53:19 +03:00
|
|
|
|
SELECT id, title, status, severity, source, ingress_event_id, created_at, updated_at,
|
|
|
|
|
|
grafana_org_slug, service_name
|
2026-04-03 09:03:16 +03:00
|
|
|
|
FROM incidents WHERE id = $1::uuid
|
|
|
|
|
|
""",
|
|
|
|
|
|
incident_id,
|
|
|
|
|
|
)
|
|
|
|
|
|
if not row:
|
|
|
|
|
|
raise HTTPException(status_code=404, detail="not found")
|
2026-04-03 13:53:19 +03:00
|
|
|
|
return _incident_row_dict(row)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@router.patch("/{incident_id}")
|
|
|
|
|
|
async def patch_incident_api(
|
|
|
|
|
|
incident_id: UUID,
|
|
|
|
|
|
body: IncidentPatch,
|
|
|
|
|
|
pool: asyncpg.Pool | None = Depends(get_pool),
|
|
|
|
|
|
):
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
raise HTTPException(status_code=503, detail="database disabled")
|
|
|
|
|
|
if body.title is None and body.status is None and body.severity is None:
|
|
|
|
|
|
raise HTTPException(status_code=400, detail="no fields to update")
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
row = await conn.fetchrow(
|
|
|
|
|
|
"""
|
|
|
|
|
|
UPDATE incidents SET
|
|
|
|
|
|
title = COALESCE($2, title),
|
|
|
|
|
|
status = COALESCE($3, status),
|
|
|
|
|
|
severity = COALESCE($4, severity),
|
|
|
|
|
|
updated_at = now()
|
|
|
|
|
|
WHERE id = $1::uuid
|
|
|
|
|
|
RETURNING id, title, status, severity, source, ingress_event_id, created_at, updated_at,
|
|
|
|
|
|
grafana_org_slug, service_name
|
|
|
|
|
|
""",
|
|
|
|
|
|
incident_id,
|
|
|
|
|
|
body.title.strip() if body.title is not None else None,
|
|
|
|
|
|
body.status,
|
|
|
|
|
|
body.severity,
|
|
|
|
|
|
)
|
|
|
|
|
|
if not row:
|
|
|
|
|
|
raise HTTPException(status_code=404, detail="not found")
|
|
|
|
|
|
return _incident_row_dict(row)
|
2026-04-03 09:03:16 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ui_router.get("/", response_class=HTMLResponse)
|
|
|
|
|
|
async def incidents_ui_home(request: Request):
|
|
|
|
|
|
pool = get_pool(request)
|
|
|
|
|
|
rows_html = ""
|
|
|
|
|
|
err = ""
|
|
|
|
|
|
if pool is None:
|
|
|
|
|
|
err = "<p>База данных не настроена.</p>"
|
|
|
|
|
|
else:
|
|
|
|
|
|
try:
|
|
|
|
|
|
async with pool.acquire() as conn:
|
|
|
|
|
|
rows = await conn.fetch(
|
|
|
|
|
|
"""
|
2026-04-03 13:53:19 +03:00
|
|
|
|
SELECT id, title, status, severity, source, created_at, grafana_org_slug, service_name
|
2026-04-03 09:03:16 +03:00
|
|
|
|
FROM incidents
|
|
|
|
|
|
ORDER BY created_at DESC
|
|
|
|
|
|
LIMIT 100
|
|
|
|
|
|
"""
|
|
|
|
|
|
)
|
|
|
|
|
|
for r in rows:
|
2026-04-03 13:53:19 +03:00
|
|
|
|
org = html.escape(str(r["grafana_org_slug"] or "—"))
|
|
|
|
|
|
svc = html.escape(str(r["service_name"] or "—"))
|
2026-04-03 09:03:16 +03:00
|
|
|
|
rows_html += (
|
|
|
|
|
|
"<tr>"
|
|
|
|
|
|
f"<td>{html.escape(str(r['id']))[:8]}…</td>"
|
|
|
|
|
|
f"<td>{html.escape(r['title'])}</td>"
|
|
|
|
|
|
f"<td>{html.escape(r['status'])}</td>"
|
|
|
|
|
|
f"<td>{html.escape(r['severity'])}</td>"
|
|
|
|
|
|
f"<td>{html.escape(r['source'])}</td>"
|
2026-04-03 13:53:19 +03:00
|
|
|
|
f"<td>{org}</td>"
|
|
|
|
|
|
f"<td>{svc}</td>"
|
2026-04-03 09:03:16 +03:00
|
|
|
|
"</tr>"
|
|
|
|
|
|
)
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
err = f"<p class=\"module-err\">{html.escape(str(e))}</p>"
|
|
|
|
|
|
inner = f"""<h1>Инциденты</h1>
|
|
|
|
|
|
{err}
|
|
|
|
|
|
<table class="irm-table">
|
2026-04-03 13:53:19 +03:00
|
|
|
|
<thead><tr><th>ID</th><th>Заголовок</th><th>Статус</th><th>Важность</th><th>Источник</th><th>Grafana slug</th><th>Сервис</th></tr></thead>
|
|
|
|
|
|
<tbody>{rows_html or '<tr><td colspan="7">Пока нет записей</td></tr>'}</tbody>
|
2026-04-03 09:03:16 +03:00
|
|
|
|
</table>
|
|
|
|
|
|
<p><small>Создание из Grafana: webhook → запись в <code>ingress_events</code> → событие → строка здесь.</small></p>"""
|
|
|
|
|
|
return HTMLResponse(
|
|
|
|
|
|
wrap_module_html_page(
|
|
|
|
|
|
document_title="Инциденты — onGuard24",
|
|
|
|
|
|
current_slug="incidents",
|
|
|
|
|
|
main_inner_html=inner,
|
|
|
|
|
|
)
|
|
|
|
|
|
)
|