diff --git a/.codex/AGENT_INSTRUCTIONS.md b/.codex/AGENT_INSTRUCTIONS.md new file mode 100644 index 0000000..15e3404 --- /dev/null +++ b/.codex/AGENT_INSTRUCTIONS.md @@ -0,0 +1,9 @@ +## Repository Operating Guide + +- Make the smallest correct change and prefer minimal diffs. +- Read `.codex/PROJECT_CONTEXT.md` before expanding into new areas. +- Preserve existing backend/frontend patterns and naming. +- Avoid whole-repo scans unless imports, failing tests, or uncertainty require it. +- Run the smallest useful validation first. +- Run tests inside containers when tests are needed. +- Do not change unrelated files or refactor opportunistically. diff --git a/.codex/PROJECT_CONTEXT.md b/.codex/PROJECT_CONTEXT.md new file mode 100644 index 0000000..918b381 --- /dev/null +++ b/.codex/PROJECT_CONTEXT.md @@ -0,0 +1,33 @@ +## Repository Context Map + +### Entry Points +- `app/main.py`: main backend API (`/api/public`, `/api/admin`). +- `app/chat_main.py`: dedicated chat API service. +- `app/email_main.py`: email service. +- `docker-compose.yml`: local service topology. + +### Main Backend Areas +- `app/api/public/`: public/client cabinet endpoints. +- `app/api/admin/`: admin and lawyer endpoints. +- `app/api/admin/requests_modules/kanban.py`: kanban aggregation and filters. +- `app/api/admin/crud_modules/`: generic CRUD/query layer. +- `app/services/`: shared domain services, including chat serialization/security. +- `app/models/`: SQLAlchemy models. +- `app/core/`: config, middleware, security hardening. + +### Frontend Areas +- `app/web/admin/`: admin/lawyer UI source modules. +- `app/web/client.jsx`: client cabinet entry. +- `app/web/admin.js`, `app/web/client.js`: built bundles. + +### Tests +- `tests/test_http_hardening.py`: middleware/security headers. +- `tests/test_public_cabinet.py`: client cabinet flows. +- `tests/admin/test_lawyer_chat.py`: admin/lawyer chat flows. +- `tests/test_migrations.py`: migration coverage. + +### High-Risk Zones +- Request workspace loading: admin `app/web/admin/hooks/useRequestWorkspace.js`, client `app/web/client.jsx`. +- Kanban performance: `app/api/admin/requests_modules/kanban.py`. +- Generic query endpoints used by request modal: `app/api/admin/crud_modules/service.py`, `app/api/admin/invoices.py`. +- Chat serialization and live updates: `app/services/chat_secure_service.py`, public/admin chat routers. diff --git a/alembic/versions/0034_add_request_assigned_lawyer_index.py b/alembic/versions/0034_add_request_assigned_lawyer_index.py new file mode 100644 index 0000000..3ead249 --- /dev/null +++ b/alembic/versions/0034_add_request_assigned_lawyer_index.py @@ -0,0 +1,37 @@ +"""add index for requests.assigned_lawyer_id + +Revision ID: 0034_request_assigned_lawyer_idx +Revises: 0033_message_receipts +Create Date: 2026-03-16 +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + + +revision = "0034_request_assigned_lawyer_idx" +down_revision = "0033_message_receipts" +branch_labels = None +depends_on = None + + +def _has_index(inspector: sa.Inspector, table: str, index_name: str) -> bool: + return any(str(idx.get("name")) == index_name for idx in inspector.get_indexes(table)) + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if not _has_index(inspector, "requests", "ix_requests_assigned_lawyer_id"): + op.create_index("ix_requests_assigned_lawyer_id", "requests", ["assigned_lawyer_id"], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + + if _has_index(inspector, "requests", "ix_requests_assigned_lawyer_id"): + op.drop_index("ix_requests_assigned_lawyer_id", table_name="requests") diff --git a/app/core/http_hardening.py b/app/core/http_hardening.py index 129ac57..4cc5fec 100644 --- a/app/core/http_hardening.py +++ b/app/core/http_hardening.py @@ -58,6 +58,23 @@ _FRAMEABLE_PATH_PATTERNS = ( re.compile(r"^/api/admin/invoices/[^/]+/pdf$"), ) +_PERF_PATH_PATTERNS = ( + ("admin_kanban", re.compile(r"^/api/admin/requests/kanban$")), + ("admin_request_detail", re.compile(r"^/api/admin/crud/requests/[^/]+$")), + ("admin_chat_messages", re.compile(r"^/api/admin/chat/requests/[^/]+/messages$")), + ("admin_chat_live", re.compile(r"^/api/admin/chat/requests/[^/]+/live$")), + ("admin_request_status_route", re.compile(r"^/api/admin/requests/[^/]+/status-route$")), + ("admin_request_attachments_query", re.compile(r"^/api/admin/crud/attachments/query$")), + ("admin_request_invoices_query", re.compile(r"^/api/admin/invoices/query$")), + ("public_request_detail", re.compile(r"^/api/public/requests/[^/]+$")), + ("public_chat_messages", re.compile(r"^/api/public/chat/requests/[^/]+/messages$")), + ("public_chat_live", re.compile(r"^/api/public/chat/requests/[^/]+/live$")), + ("public_request_attachments", re.compile(r"^/api/public/requests/[^/]+/attachments$")), + ("public_request_invoices", re.compile(r"^/api/public/requests/[^/]+/invoices$")), + ("public_request_status_route", re.compile(r"^/api/public/requests/[^/]+/status-route$")), + ("public_request_service_requests", re.compile(r"^/api/public/requests/[^/]+/service-requests$")), +) + def _request_id_from_header(raw: str | None) -> str: value = str(raw or "").strip() @@ -76,6 +93,17 @@ def _response_security_headers(request: Request) -> dict[str, str]: return SECURITY_HEADERS +def _performance_label(request: Request) -> str | None: + method = str(request.method or "").upper() + if method not in {"GET", "POST"}: + return None + path = str(request.url.path or "") + for label, pattern in _PERF_PATH_PATTERNS: + if pattern.search(path): + return label + return None + + def install_http_hardening(app: FastAPI) -> None: @app.middleware("http") async def _http_hardening_middleware(request: Request, call_next): @@ -95,12 +123,18 @@ def install_http_hardening(app: FastAPI) -> None: response.headers[REQUEST_ID_HEADER] = request_id duration_ms = (perf_counter() - started_at) * 1000.0 + perf_label = _performance_label(request) + if perf_label: + response.headers["Server-Timing"] = f'app;desc="{perf_label}";dur={duration_ms:.2f}' + response.headers["X-Perf-Label"] = perf_label + response.headers["X-Perf-Duration-Ms"] = f"{duration_ms:.2f}" _LOG.info( - "%s %s status=%s duration_ms=%.2f request_id=%s", + "%s %s status=%s duration_ms=%.2f request_id=%s perf_label=%s", request.method, request.url.path, response.status_code, duration_ms, request_id, + perf_label or "-", ) return response diff --git a/app/models/request.py b/app/models/request.py index 308bcb7..6632d95 100644 --- a/app/models/request.py +++ b/app/models/request.py @@ -22,7 +22,7 @@ class Request(Base, UUIDMixin, TimestampMixin): important_date_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True) description: Mapped[str | None] = mapped_column(Text, nullable=True) extra_fields: Mapped[dict] = mapped_column(JSON, default=dict, nullable=False) - assigned_lawyer_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + assigned_lawyer_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) effective_rate: Mapped[float | None] = mapped_column(Numeric(12, 2), nullable=True) request_cost: Mapped[float | None] = mapped_column(Numeric(14, 2), nullable=True) invoice_amount: Mapped[float | None] = mapped_column(Numeric(14, 2), nullable=True) diff --git a/context/11_test_runbook.md b/context/11_test_runbook.md index e4fba9a..c78d788 100644 --- a/context/11_test_runbook.md +++ b/context/11_test_runbook.md @@ -58,6 +58,11 @@ curl -fsS http://localhost:8081/chat-health ./scripts/ops/check_chat_health.sh echo $? # 0=OK, >0=ALERT ``` +10. Снятие baseline по производительности admin workspace: +```bash +./scripts/ops/perf_baseline.sh http://localhost:8081 +``` +Отчет сохраняется в `reports/perf/perf-baseline-.md`. Скрипт логинится под `admin@example.com / admin123`, берет первую заявку из канбана и замеряет `kanban`, `request detail`, `chat messages/live`, `status-route`, `attachments`, `invoices`. ## Матрица проверок по задачам | ID | Что проверяем | Где тесты | Как запускать | diff --git a/context/19_performance_tracking_2026-03-16.md b/context/19_performance_tracking_2026-03-16.md new file mode 100644 index 0000000..d897b6c --- /dev/null +++ b/context/19_performance_tracking_2026-03-16.md @@ -0,0 +1,55 @@ +# Performance Tracking + +Дата старта: 2026-03-16 + +## Цель + +Снизить воспринимаемую задержку при открытии канбана, карточки заявки и чата. Базовая инфраструктура: `4 vCPU / 8 GB RAM / SSD 150 GB`. + +## Текущая гипотеза + +- Основная задержка создается не железом, а лишними round-trip между фронтом и backend/chat-service. +- Карточка заявки загружается несколькими параллельными запросами и при live-обновлениях перезагружается целиком. +- Канбан собирается через загрузку большого массива заявок в Python и дальнейшую агрегацию в памяти. + +## Backlog + +| ID | Задача | Статус | Приоритет | Зависимости | +|---|---|---|---|---| +| PERF-01 | Зафиксировать baseline по ключевым endpoint и сценариям | in_progress | P0 | — | +| PERF-02 | Добавить индекс на `requests.assigned_lawyer_id` | completed | P0 | — | +| PERF-03 | Убрать full reload карточки заявки при live-обновлениях | planned | P0 | PERF-01 | +| PERF-04 | Собрать единый endpoint карточки заявки | planned | P0 | PERF-01 | +| PERF-05 | Выделить узкие request-scoped endpoints для вложений и счетов | planned | P0 | PERF-04 | +| PERF-06 | Переписать kanban на SQL-first фильтрацию/limit | planned | P0 | PERF-01, PERF-02 | +| PERF-07 | Ограничить initial chat payload и добавить догрузку истории | planned | P1 | PERF-03, PERF-04 | +| PERF-08 | Добавить нужные вспомогательные индексы и повторный profiling | planned | P1 | PERF-01 | + +## PERF-01 + +### Scope + +- Добавить серверные замеры для проблемных endpoint без изменения контрактов. +- Помечать в логах и headers целевые сценарии: + - kanban + - request workspace + - chat messages + - chat live + - status route + - attachments + - invoices + +### Progress + +- 2026-03-16: создан tracking-файл. +- 2026-03-16: в работе точечная инструментализация через существующий HTTP middleware. +- 2026-03-16: добавлены `Server-Timing`, `X-Perf-Label`, `X-Perf-Duration-Ms` для ключевых endpoint. +- 2026-03-16: контейнерный тест `python -m unittest tests.test_http_hardening -v` пройден. +- 2026-03-16: добавлен ops-скрипт `scripts/ops/perf_baseline.sh` для repeatable baseline по admin workspace. +- 2026-03-16: baseline еще не снят, потому что локальный контур на `localhost:8081` не поднят. +- 2026-03-16: выполнен `PERF-02` - добавлен индекс `ix_requests_assigned_lawyer_id`, миграционный тест пройден. + +## Дальше + +1. Поднять локальный контур и выполнить `./scripts/ops/perf_baseline.sh http://localhost:8081`. +2. После снятия baseline перейти к `PERF-06` и убирать `base_query.all()` из kanban. diff --git a/scripts/ops/perf_baseline.sh b/scripts/ops/perf_baseline.sh new file mode 100755 index 0000000..3262917 --- /dev/null +++ b/scripts/ops/perf_baseline.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +BASE_URL="${1:-http://localhost:8081}" +REPORT_DIR="${REPORT_DIR:-reports/perf}" +ITERATIONS="${PERF_ITERATIONS:-5}" +ADMIN_EMAIL="${PERF_ADMIN_EMAIL:-admin@example.com}" +ADMIN_PASSWORD="${PERF_ADMIN_PASSWORD:-admin123}" +KANBAN_LIMIT="${PERF_KANBAN_LIMIT:-400}" +TS_HUMAN="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" +TS_FILE="$(date -u +"%Y%m%d-%H%M%S")" +REPORT_FILE="${REPORT_DIR}/perf-baseline-${TS_FILE}.md" + +mkdir -p "$REPORT_DIR" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing command: $1" >&2 + exit 1 + } +} + +require_cmd curl +require_cmd python3 + +json_escape() { + python3 - "$1" <<'PY' +import json +import sys +print(json.dumps(sys.argv[1])) +PY +} + +LOGIN_BODY="$(printf '{"email":%s,"password":%s}' "$(json_escape "$ADMIN_EMAIL")" "$(json_escape "$ADMIN_PASSWORD")")" +LOGIN_RESPONSE_FILE="$TMP_DIR/login.json" + +curl -fsS \ + -H "Content-Type: application/json" \ + -X POST \ + -d "$LOGIN_BODY" \ + "$BASE_URL/api/admin/auth/login" >"$LOGIN_RESPONSE_FILE" + +AUTH_TOKEN="$(python3 - "$LOGIN_RESPONSE_FILE" <<'PY' +import json +import sys +with open(sys.argv[1], "r", encoding="utf-8") as fh: + data = json.load(fh) +token = str(data.get("access_token") or "").strip() +if not token: + raise SystemExit("login did not return access_token") +print(token) +PY +)" + +KANBAN_BODY_FILE="$TMP_DIR/kanban.json" +curl -fsS \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + "$BASE_URL/api/admin/requests/kanban?limit=${KANBAN_LIMIT}&sort_mode=created_newest" >"$KANBAN_BODY_FILE" + +REQUEST_ID="$(python3 - "$KANBAN_BODY_FILE" <<'PY' +import json +import sys +with open(sys.argv[1], "r", encoding="utf-8") as fh: + data = json.load(fh) +rows = data.get("rows") or [] +if not rows: + raise SystemExit("kanban returned no rows; seed manual data first") +request_id = str((rows[0] or {}).get("id") or "").strip() +if not request_id: + raise SystemExit("kanban first row has no id") +print(request_id) +PY +)" + +ATTACHMENTS_BODY="$(printf '{"filters":[{"field":"request_id","op":"=","value":%s}],"sort":[{"field":"created_at","dir":"asc"}],"page":{"limit":500,"offset":0}}' "$(json_escape "$REQUEST_ID")")" +INVOICES_BODY="$(printf '{"filters":[{"field":"request_id","op":"=","value":%s}],"sort":[{"field":"issued_at","dir":"desc"}],"page":{"limit":500,"offset":0}}' "$(json_escape "$REQUEST_ID")")" + +measure_endpoint() { + local name="$1" + local method="$2" + local path="$3" + local body="${4:-}" + + local headers_file body_file curl_meta status_code total_ms perf_label perf_duration + for run in $(seq 1 "$ITERATIONS"); do + headers_file="$TMP_DIR/${name}-${run}.headers" + body_file="$TMP_DIR/${name}-${run}.body" + if [[ "$method" == "POST" ]]; then + curl_meta="$(curl -sS \ + -D "$headers_file" \ + -o "$body_file" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "$body" \ + -w '%{http_code} %{time_total}' \ + "$BASE_URL$path")" + else + curl_meta="$(curl -sS \ + -D "$headers_file" \ + -o "$body_file" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -w '%{http_code} %{time_total}' \ + "$BASE_URL$path")" + fi + + status_code="$(echo "$curl_meta" | awk '{print $1}')" + total_ms="$(echo "$curl_meta" | awk '{printf "%.2f", $2 * 1000}')" + + if [[ "$status_code" != "200" ]]; then + echo "endpoint ${name} failed: HTTP ${status_code}" >&2 + cat "$body_file" >&2 || true + exit 1 + fi + + python3 - "$headers_file" "$name" "$run" "$total_ms" >>"$TMP_DIR/raw.tsv" <<'PY' +import sys + +headers_path, name, run, total_ms = sys.argv[1:5] +headers = {} +with open(headers_path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or ":" not in line: + continue + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() +print("\t".join([ + name, + run, + total_ms, + headers.get("x-perf-label", ""), + headers.get("x-perf-duration-ms", ""), +])) +PY + done +} + +: >"$TMP_DIR/raw.tsv" + +measure_endpoint "kanban" "GET" "/api/admin/requests/kanban?limit=${KANBAN_LIMIT}&sort_mode=created_newest" +measure_endpoint "request_detail" "GET" "/api/admin/crud/requests/${REQUEST_ID}" +measure_endpoint "chat_messages" "GET" "/api/admin/chat/requests/${REQUEST_ID}/messages" +measure_endpoint "chat_live" "GET" "/api/admin/chat/requests/${REQUEST_ID}/live" +measure_endpoint "status_route" "GET" "/api/admin/requests/${REQUEST_ID}/status-route" +measure_endpoint "attachments_query" "POST" "/api/admin/crud/attachments/query" "$ATTACHMENTS_BODY" +measure_endpoint "invoices_query" "POST" "/api/admin/invoices/query" "$INVOICES_BODY" + +python3 - "$TMP_DIR/raw.tsv" "$REPORT_FILE" "$TS_HUMAN" "$BASE_URL" "$REQUEST_ID" "$ITERATIONS" <<'PY' +import csv +import statistics +import sys +from collections import defaultdict + +raw_path, report_path, ts_human, base_url, request_id, iterations = sys.argv[1:7] + +rows = defaultdict(list) +with open(raw_path, "r", encoding="utf-8") as fh: + reader = csv.reader(fh, delimiter="\t") + for name, run, total_ms, perf_label, perf_duration in reader: + rows[name].append( + { + "run": int(run), + "total_ms": float(total_ms or 0), + "perf_label": perf_label or "-", + "perf_duration_ms": float(perf_duration or 0), + } + ) + +def percentile(sorted_values, ratio): + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return sorted_values[0] + index = round((len(sorted_values) - 1) * ratio) + return sorted_values[index] + +with open(report_path, "w", encoding="utf-8") as out: + out.write("# Perf Baseline Report\n\n") + out.write(f"- Timestamp: `{ts_human}`\n") + out.write(f"- Base URL: `{base_url}`\n") + out.write(f"- Request ID sample: `{request_id}`\n") + out.write(f"- Iterations per endpoint: `{iterations}`\n\n") + out.write("| Endpoint | Perf Label | Avg Total ms | P95 Total ms | Avg Server ms |\n") + out.write("|---|---|---:|---:|---:|\n") + for name in [ + "kanban", + "request_detail", + "chat_messages", + "chat_live", + "status_route", + "attachments_query", + "invoices_query", + ]: + items = rows.get(name, []) + totals = sorted(item["total_ms"] for item in items) + servers = [item["perf_duration_ms"] for item in items if item["perf_duration_ms"] > 0] + avg_total = statistics.mean(totals) if totals else 0.0 + p95_total = percentile(totals, 0.95) + avg_server = statistics.mean(servers) if servers else 0.0 + label = items[0]["perf_label"] if items else "-" + out.write(f"| {name} | `{label}` | {avg_total:.2f} | {p95_total:.2f} | {avg_server:.2f} |\n") + out.write("\n## Raw Runs\n\n") + out.write("| Endpoint | Run | Total ms | Server ms |\n") + out.write("|---|---:|---:|---:|\n") + for name, items in rows.items(): + for item in sorted(items, key=lambda value: value["run"]): + out.write(f"| {name} | {item['run']} | {item['total_ms']:.2f} | {item['perf_duration_ms']:.2f} |\n") +PY + +echo "report: $REPORT_FILE" diff --git a/tests/test_http_hardening.py b/tests/test_http_hardening.py index 8b25e24..88e7dc6 100644 --- a/tests/test_http_hardening.py +++ b/tests/test_http_hardening.py @@ -11,7 +11,7 @@ os.environ.setdefault("S3_SECRET_KEY", "test") os.environ.setdefault("S3_BUCKET", "test") from app.main import app -from app.core.http_hardening import _response_security_headers +from app.core.http_hardening import _performance_label, _response_security_headers from starlette.requests import Request @@ -77,6 +77,35 @@ class HttpHardeningTests(unittest.TestCase): self.assertEqual(headers.get("X-Frame-Options"), "SAMEORIGIN") self.assertIn("frame-ancestors 'self'", str(headers.get("Content-Security-Policy"))) + def test_target_perf_endpoint_has_observability_headers(self): + response = self.client.get("/api/admin/requests/kanban") + self.assertEqual(response.status_code, 401) + self.assertEqual(response.headers.get("x-perf-label"), "admin_kanban") + self.assertTrue(bool(response.headers.get("x-perf-duration-ms"))) + self.assertIn('desc="admin_kanban"', str(response.headers.get("server-timing"))) + + def test_non_target_endpoint_has_no_perf_headers(self): + response = self.client.get("/health") + self.assertEqual(response.status_code, 200) + self.assertIsNone(response.headers.get("x-perf-label")) + self.assertIsNone(response.headers.get("x-perf-duration-ms")) + self.assertIsNone(response.headers.get("server-timing")) + + def test_performance_label_maps_client_workspace_endpoints(self): + scope = { + "type": "http", + "http_version": "1.1", + "method": "GET", + "scheme": "http", + "path": "/api/public/requests/TRK-1/status-route", + "raw_path": b"/api/public/requests/TRK-1/status-route", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 12345), + "server": ("testserver", 80), + } + self.assertEqual(_performance_label(Request(scope)), "public_request_status_route") + def test_non_file_paths_keep_deny_framing(self): scope = { "type": "http", diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 5fbb441..cac08e4 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -114,7 +114,7 @@ class MigrationTests(unittest.TestCase): def test_alembic_version_is_set(self): with self.engine.connect() as conn: version = conn.execute(text("SELECT version_num FROM alembic_version")).scalar_one() - self.assertEqual(version, "0033_message_receipts") + self.assertEqual(version, "0034_request_assigned_lawyer_idx") def test_responsible_column_exists_in_all_domain_tables(self): tables = { @@ -205,6 +205,10 @@ class MigrationTests(unittest.TestCase): self.assertIn("paid_at", columns) self.assertIn("paid_by_admin_id", columns) + def test_requests_contains_assigned_lawyer_index(self): + indexes = {index["name"] for index in self.inspector.get_indexes("requests")} + self.assertIn("ix_requests_assigned_lawyer_id", indexes) + def test_data_retention_policies_contains_core_columns(self): columns = {column["name"] for column in self.inspector.get_columns("data_retention_policies")} self.assertIn("id", columns)