From 2b7043a89ed6e94d7651040c85e904e6893dba97 Mon Sep 17 00:00:00 2001 From: TronoSfera <119615520+TronoSfera@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:30:43 +0300 Subject: [PATCH] fix speed up 04 --- .codex/PROJECT_CONTEXT.md | 4 + Makefile | 13 +- app/api/admin/chat.py | 107 +++++++++- app/api/admin/requests_modules/router.py | 3 +- app/api/admin/requests_modules/service.py | 102 ++++++++-- app/api/admin/requests_modules/status_flow.py | 33 ++- app/api/public/chat.py | 107 +++++++++- app/core/http_hardening.py | 2 + app/models/message.py | 45 ++++- app/scripts/reencrypt_with_active_kid.py | 38 +++- app/services/chat_crypto.py | 190 ++++++++++++++++-- app/services/chat_secure_service.py | 184 +++++++++++++++-- app/web/admin.js | 154 ++++++++++---- .../features/requests/RequestWorkspace.jsx | 4 +- app/web/admin/hooks/useRequestWorkspace.js | 163 +++++++++++---- app/web/client.js | 151 +++++++++++++- app/web/client.jsx | 91 +++++++-- context/19_performance_tracking_2026-03-16.md | 13 +- requirements.txt | 1 + tests/admin/test_lawyer_chat.py | 38 +++- tests/test_crypto_kid_rotation.py | 26 ++- tests/test_invoices.py | 6 +- tests/test_public_cabinet.py | 32 ++- tests/test_reencrypt_with_active_kid.py | 5 +- 24 files changed, 1309 insertions(+), 203 deletions(-) diff --git a/.codex/PROJECT_CONTEXT.md b/.codex/PROJECT_CONTEXT.md index 918b381..c2491ef 100644 --- a/.codex/PROJECT_CONTEXT.md +++ b/.codex/PROJECT_CONTEXT.md @@ -12,7 +12,10 @@ - `app/api/admin/requests_modules/kanban.py`: kanban aggregation and filters. - `app/api/admin/crud_modules/`: generic CRUD/query layer. - `app/services/`: shared domain services, including chat serialization/security. + - `app/services/chat_crypto.py`: versioned chat crypto (`v1/v2` backward-compatible read, `v3` per-chat AEAD write path). + - `app/services/chat_secure_service.py`: chat paging, explicit decrypt, message body batches, live delta. - `app/models/`: SQLAlchemy models. + - `app/models/message.py`: plaintext no longer auto-decrypts from ORM; chat body encryption happens on flush, read decrypt is explicit. - `app/core/`: config, middleware, security hardening. ### Frontend Areas @@ -31,3 +34,4 @@ - Kanban performance: `app/api/admin/requests_modules/kanban.py`. - Generic query endpoints used by request modal: `app/api/admin/crud_modules/service.py`, `app/api/admin/invoices.py`. - Chat serialization and live updates: `app/services/chat_secure_service.py`, public/admin chat routers. +- Chat crypto and migration safety: `app/services/chat_crypto.py`, `app/scripts/reencrypt_with_active_kid.py`, `tests/test_crypto_kid_rotation.py`, `tests/test_reencrypt_with_active_kid.py`. diff --git a/Makefile b/Makefile index 4925049..7fd1a5f 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ .PHONY: \ help \ local-up local-down local-logs local-migrate local-test local-seed local-seed-statuses local-seed-catalog \ + local-reencrypt-active-kid \ prod-up prod-down prod-logs prod-ps prod-migrate \ prod-seed-statuses prod-seed-catalog \ prod-secrets-generate prod-secrets-apply prod-secrets-generate-env prod-secrets-apply-env \ - prod-minio-tls-init incident-checklist rotate-encryption-kid reencrypt-active-kid \ + prod-minio-tls-init incident-checklist rotate-encryption-kid reencrypt-active-kid prod-reencrypt-active-kid \ security-smoke prod-security-audit prod-security-scheduler-up prod-security-scheduler-logs \ prod-cert-init prod-cert-renew \ check-prod-files check-cert-files \ @@ -37,6 +38,7 @@ help: @echo " local-seed - Seed quotes (local)" @echo " local-seed-statuses - Seed legal flow statuses (local)" @echo " local-seed-catalog - Seed quotes + legal flow statuses (local)" + @echo " local-reencrypt-active-kid - Re-encrypt historical chat/invoice/admin secrets using active KID (local)" @echo " prod-up - Start production stack (nginx 80/443 + TLS certs already issued)" @echo " prod-down - Stop production stack" @echo " prod-logs - Tail production logs" @@ -48,6 +50,7 @@ help: @echo " prod-secrets-apply - Generate + apply rotated internal secrets to running prod stack" @echo " prod-secrets-generate-env - Generate rotated secrets from current .env into .env.secure" @echo " prod-secrets-apply-env - Generate + apply rotated secrets directly for current .env" + @echo " prod-reencrypt-active-kid - Re-encrypt historical chat/invoice/admin secrets using active KID (prod)" @echo " prod-minio-tls-init - Generate internal CA and MinIO TLS certs (deploy/tls/minio)" @echo " incident-checklist - Create PDn incident checklist markdown report" @echo " security-smoke - Run security smoke checks and create report" @@ -95,6 +98,9 @@ local-seed-catalog: $(LOCAL_COMPOSE) exec -T backend python -m app.scripts.upsert_quotes $(LOCAL_COMPOSE) exec -T backend python -m app.scripts.upsert_statuses_legal_flow +local-reencrypt-active-kid: + $(LOCAL_COMPOSE) exec -T backend python -m app.scripts.reencrypt_with_active_kid --apply + check-prod-files: @test -f docker-compose.prod.nginx.yml || (echo "[ERROR] Missing docker-compose.prod.nginx.yml. Run: git pull"; exit 1) @test -f frontend/nginx.prod.conf || (echo "[ERROR] Missing frontend/nginx.prod.conf. Run: git pull"; exit 1) @@ -177,7 +183,10 @@ rotate-encryption-kid: ./scripts/ops/rotate_encryption_kid.sh --env-file .env reencrypt-active-kid: - docker compose exec -T backend python -m app.scripts.reencrypt_with_active_kid --apply + $(MAKE) local-reencrypt-active-kid + +prod-reencrypt-active-kid: check-prod-files + $(PROD_COMPOSE) exec -T backend python -m app.scripts.reencrypt_with_active_kid --apply # Initial certificate bootstrap: # 1) Start stack with edge nginx on port 80 only. diff --git a/app/api/admin/chat.py b/app/api/admin/chat.py index 4f74e2a..1b18c09 100644 --- a/app/api/admin/chat.py +++ b/app/api/admin/chat.py @@ -28,7 +28,8 @@ from app.services.chat_secure_service import ( list_messages_for_request, mark_messages_delivered_for_staff, mark_messages_read_for_staff, - serialize_message, + serialize_message_for_request, + serialize_message_bodies_for_request, serialize_messages_for_request, ) from app.services.chat_presence import list_typing_presence, set_typing_presence @@ -142,6 +143,24 @@ def _parse_uuid_or_400(raw: str, field_name: str) -> UUID: raise HTTPException(status_code=400, detail=f'Некорректное поле "{field_name}"') +def _normalize_message_ids(raw: object, *, field_name: str = "ids", limit: int = 200) -> list[UUID]: + if not isinstance(raw, list): + raise HTTPException(status_code=400, detail=f'Поле "{field_name}" должно быть списком') + seen: set[UUID] = set() + out: list[UUID] = [] + for item in raw: + value = _parse_uuid_or_400(str(item or ""), field_name) + if value in seen: + continue + seen.add(value) + out.append(value) + if len(out) >= limit: + break + if not out: + raise HTTPException(status_code=400, detail="Нужно передать хотя бы один идентификатор сообщения") + return out + + def _slugify_key(raw: str) -> str: text = str(raw or "").strip().lower() out = [] @@ -276,7 +295,16 @@ def list_request_messages( _ensure_lawyer_can_view_request_or_403(admin, req) mark_messages_read_for_staff(db, request_id=req.id) rows = list_messages_for_request(db, req.id) - payload = {"rows": serialize_messages_for_request(db, req.id, rows), "total": len(rows)} + payload = { + "rows": serialize_messages_for_request( + db, + req.id, + rows, + request_extra_fields=req.extra_fields, + include_bodies=True, + ), + "total": len(rows), + } _audit_admin_chat_read( db, admin=admin, @@ -292,25 +320,36 @@ def list_request_messages( def list_request_messages_window( request_id: str, http_request: FastapiRequest, + before_id: str | None = None, + before_created_at: str | None = None, before_count: int = 0, limit: int = DEFAULT_CHAT_WINDOW_LIMIT, + include_body: bool = True, db: Session = Depends(get_db), admin: dict = Depends(require_role("ADMIN", "LAWYER", "CURATOR")), ): req = _request_for_id_or_404(db, request_id) _ensure_lawyer_can_view_request_or_403(admin, req) mark_messages_read_for_staff(db, request_id=req.id) - rows, total, has_more, loaded_count = list_messages_for_request_window( + rows, has_more = list_messages_for_request_window( db, req.id, limit=limit, + before_id=before_id, + before_created_at=before_created_at, before_count=before_count, ) + message_total = int(get_chat_activity_summary(db, req.id).get("message_count") or len(rows)) payload = { - "rows": serialize_messages_for_request(db, req.id, rows), - "total": total, + "rows": serialize_messages_for_request( + db, + req.id, + rows, + request_extra_fields=req.extra_fields, + include_bodies=bool(include_body), + ), "has_more": has_more, - "loaded_count": loaded_count, + "total": message_total, "limit": clamp_chat_window_limit(limit), } _audit_admin_chat_read( @@ -324,6 +363,44 @@ def list_request_messages_window( return payload +@router.post("/requests/{request_id}/message-bodies") +def load_request_message_bodies( + request_id: str, + payload: dict, + http_request: FastapiRequest, + db: Session = Depends(get_db), + admin: dict = Depends(require_role("ADMIN", "LAWYER", "CURATOR")), +): + req = _request_for_id_or_404(db, request_id) + _ensure_lawyer_can_view_request_or_403(admin, req) + message_ids = _normalize_message_ids((payload or {}).get("ids")) + rows = ( + db.query(Message) + .filter(Message.request_id == req.id, Message.id.in_(message_ids)) + .order_by(Message.created_at.asc(), Message.id.asc()) + .all() + ) + rows_by_id = {str(row.id): row for row in rows} + ordered_rows = [rows_by_id[str(message_id)] for message_id in message_ids if str(message_id) in rows_by_id] + result = { + "rows": serialize_message_bodies_for_request( + db, + req.id, + ordered_rows, + request_extra_fields=req.extra_fields, + ) + } + _audit_admin_chat_read( + db, + admin=admin, + http_request=http_request, + req=req, + action="READ_CHAT_MESSAGES", + details={"rows": len(ordered_rows), "body_batch": True}, + ) + return result + + @router.post("/requests/{request_id}/messages", status_code=201) def create_request_message( request_id: str, @@ -354,7 +431,7 @@ def create_request_message( actor_name=actor_name, actor_admin_user_id=actor_admin_user_id, ) - return serialize_message(row) + return serialize_message_for_request(row, request_extra_fields=req.extra_fields) @router.get("/requests/{request_id}/live") @@ -394,7 +471,13 @@ def get_request_live_state( .order_by(Attachment.created_at.asc(), Attachment.id.asc()) .all() ) - delta_messages = serialize_messages_for_request(db, req.id, message_rows) + delta_messages = serialize_messages_for_request( + db, + req.id, + message_rows, + request_extra_fields=req.extra_fields, + include_bodies=True, + ) delta_attachments = [_serialize_live_attachment(row) for row in attachment_rows] actor_sub = str(admin.get("sub") or "").strip() or "unknown" @@ -935,7 +1018,13 @@ def upsert_data_request_batch( db.commit() fresh_messages = list_messages_for_request(db, req.id) - serialized = serialize_messages_for_request(db, req.id, fresh_messages) + serialized = serialize_messages_for_request( + db, + req.id, + fresh_messages, + request_extra_fields=req.extra_fields, + include_bodies=True, + ) payload_row = next((item for item in serialized if str(item.get("id")) == str(message_uuid)), None) if payload_row is None: raise HTTPException(status_code=500, detail="Не удалось сформировать сообщение запроса") diff --git a/app/api/admin/requests_modules/router.py b/app/api/admin/requests_modules/router.py index 6863b1b..eda2bc8 100644 --- a/app/api/admin/requests_modules/router.py +++ b/app/api/admin/requests_modules/router.py @@ -108,10 +108,11 @@ def get_request( def get_request_workspace( request_id: str, http_request: FastapiRequest, + include_related: bool = Query(default=True), db: Session = Depends(get_db), admin=Depends(require_role("ADMIN", "LAWYER", "CURATOR")), ): - payload = get_request_workspace_service(request_id, db, admin) + payload = get_request_workspace_service(request_id, db, admin, include_related=include_related) request_payload = payload.get("request") or {} record_pii_access_event( db, diff --git a/app/api/admin/requests_modules/service.py b/app/api/admin/requests_modules/service.py index 4c61a8b..39e7a1f 100644 --- a/app/api/admin/requests_modules/service.py +++ b/app/api/admin/requests_modules/service.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging +from time import perf_counter from datetime import datetime, timezone from typing import Any from uuid import UUID, uuid4 @@ -18,7 +20,7 @@ from app.models.request import Request from app.models.request_service_request import RequestServiceRequest from app.schemas.admin import RequestAdminCreate, RequestAdminPatch from app.services.chat_secure_service import ( - DEFAULT_CHAT_WINDOW_LIMIT, + get_chat_activity_summary, list_messages_for_request_window, mark_messages_read_for_staff, serialize_messages_for_request, @@ -54,6 +56,9 @@ from .permissions import ( ) from .status_flow import apply_request_special_filters, get_request_status_route_service, split_request_special_filters +_WORKSPACE_LOG = logging.getLogger("uvicorn.error") +INITIAL_WORKSPACE_CHAT_WINDOW_LIMIT = 20 + def query_requests_service(uq: UniversalQuery, db: Session, admin: dict) -> dict[str, Any]: base_query = db.query(Request) @@ -459,36 +464,76 @@ def _serialize_request_invoice(row: Invoice) -> dict[str, Any]: } -def get_request_workspace_service(request_id: str, db: Session, admin: dict) -> dict[str, Any]: +def get_request_workspace_service(request_id: str, db: Session, admin: dict, *, include_related: bool = True) -> dict[str, Any]: + started_at = perf_counter() request_uuid = request_uuid_or_400(request_id) req = db.get(Request, request_uuid) if req is None: raise HTTPException(status_code=404, detail="Заявка не найдена") ensure_lawyer_can_view_request_or_403(admin, req) + side_effects_started_at = perf_counter() _apply_request_open_side_effects(db, req, admin, mark_chat_read=True) + side_effects_ms = (perf_counter() - side_effects_started_at) * 1000.0 + + serialize_request_started_at = perf_counter() request_payload = _serialize_request_row(req) - message_rows, messages_total, messages_has_more, messages_loaded_count = list_messages_for_request_window( + request_row_ms = (perf_counter() - serialize_request_started_at) * 1000.0 + + messages_started_at = perf_counter() + message_rows, messages_has_more = list_messages_for_request_window( db, req.id, - limit=DEFAULT_CHAT_WINDOW_LIMIT, + limit=INITIAL_WORKSPACE_CHAT_WINDOW_LIMIT, before_count=0, ) - attachment_rows = ( - db.query(Attachment) - .filter(Attachment.request_id == req.id) - .order_by(Attachment.created_at.asc(), Attachment.id.asc()) - .all() + messages_query_ms = (perf_counter() - messages_started_at) * 1000.0 + + serialize_messages_started_at = perf_counter() + serialized_messages = serialize_messages_for_request( + db, + req.id, + message_rows, + request_extra_fields=req.extra_fields, + include_bodies=False, ) - role = str(admin.get("role") or "").upper() + serialize_messages_ms = (perf_counter() - serialize_messages_started_at) * 1000.0 + messages_total = int(get_chat_activity_summary(db, req.id).get("message_count") or len(message_rows)) + messages_loaded_count = len(message_rows) + + attachment_rows: list[Attachment] = [] invoice_rows: list[Invoice] = [] - if role in {"ADMIN", "LAWYER"}: - invoice_rows = ( - db.query(Invoice) - .filter(Invoice.request_id == req.id) - .order_by(Invoice.issued_at.desc(), Invoice.id.desc()) + status_route_payload: dict[str, Any] = { + "nodes": [], + "history": [], + "available_statuses": [], + "current_important_date_at": request_payload.get("important_date_at"), + } + attachments_query_ms = 0.0 + invoices_query_ms = 0.0 + status_route_ms = 0.0 + if include_related: + attachments_started_at = perf_counter() + attachment_rows = ( + db.query(Attachment) + .filter(Attachment.request_id == req.id) + .order_by(Attachment.created_at.asc(), Attachment.id.asc()) .all() ) + attachments_query_ms = (perf_counter() - attachments_started_at) * 1000.0 + role = str(admin.get("role") or "").upper() + if role in {"ADMIN", "LAWYER"}: + invoices_started_at = perf_counter() + invoice_rows = ( + db.query(Invoice) + .filter(Invoice.request_id == req.id) + .order_by(Invoice.issued_at.desc(), Invoice.id.desc()) + .all() + ) + invoices_query_ms = (perf_counter() - invoices_started_at) * 1000.0 + status_route_started_at = perf_counter() + status_route_payload = get_request_status_route_service(request_id, db, admin, request_row=req) + status_route_ms = (perf_counter() - status_route_started_at) * 1000.0 paid_invoices = [row for row in invoice_rows if str(row.status or "").upper() == "PAID"] paid_total = round(sum(float(row.amount or 0) for row in paid_invoices), 2) @@ -499,9 +544,9 @@ def get_request_workspace_service(request_id: str, db: Session, admin: dict) -> if latest_paid_at is None or row.paid_at > latest_paid_at: latest_paid_at = row.paid_at - return { + payload = { "request": request_payload, - "messages": serialize_messages_for_request(db, req.id, message_rows), + "messages": serialized_messages, "messages_total": messages_total, "messages_has_more": messages_has_more, "messages_loaded_count": messages_loaded_count, @@ -513,8 +558,29 @@ def get_request_workspace_service(request_id: str, db: Session, admin: dict) -> "paid_total": paid_total, "last_paid_at": latest_paid_at.isoformat() if latest_paid_at else request_payload.get("paid_at"), }, - "status_route": get_request_status_route_service(request_id, db, admin, request_row=req), + "status_route": status_route_payload, } + total_ms = (perf_counter() - started_at) * 1000.0 + _WORKSPACE_LOG.info( + "workspace request_id=%s include_related=%s total_ms=%.2f side_effects_ms=%.2f request_row_ms=%.2f " + "messages_query_ms=%.2f serialize_messages_ms=%.2f attachments_query_ms=%.2f invoices_query_ms=%.2f " + "status_route_ms=%.2f messages=%s attachments=%s invoices=%s messages_total=%s", + str(req.id), + bool(include_related), + total_ms, + side_effects_ms, + request_row_ms, + messages_query_ms, + serialize_messages_ms, + attachments_query_ms, + invoices_query_ms, + status_route_ms, + len(message_rows), + len(attachment_rows), + len(invoice_rows), + messages_total, + ) + return payload def claim_request_service(request_id: str, db: Session, admin: dict) -> dict[str, Any]: diff --git a/app/api/admin/requests_modules/status_flow.py b/app/api/admin/requests_modules/status_flow.py index b9cff4e..55474ec 100644 --- a/app/api/admin/requests_modules/status_flow.py +++ b/app/api/admin/requests_modules/status_flow.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging +from time import perf_counter from datetime import datetime, timedelta, timezone from typing import Any from uuid import UUID @@ -29,6 +31,8 @@ from app.services.status_transition_requirements import validate_transition_requ from .common import normalize_important_date_or_default, parse_datetime_safe from .permissions import ensure_lawyer_can_manage_request_or_403, ensure_lawyer_can_view_request_or_403, request_uuid_or_400 +_STATUS_ROUTE_LOG = logging.getLogger("uvicorn.error") + def terminal_status_codes(db: Session) -> set[str]: rows = db.query(Status.code).filter(Status.is_terminal.is_(True)).all() @@ -215,6 +219,7 @@ def get_request_status_route_service( admin: dict, request_row: Request | None = None, ) -> dict[str, Any]: + started_at = perf_counter() req = request_row if req is None: request_uuid = request_uuid_or_400(request_id) @@ -226,12 +231,14 @@ def get_request_status_route_service( topic_code = str(req.topic_code or "").strip() current_status = str(req.status_code or "").strip() + history_started_at = perf_counter() history_rows = ( db.query(StatusHistory) .filter(StatusHistory.request_id == req.id) .order_by(StatusHistory.created_at.asc()) .all() ) + history_ms = (perf_counter() - history_started_at) * 1000.0 known_codes: set[str] = set() if current_status: @@ -244,23 +251,27 @@ def get_request_status_route_service( if to_code: known_codes.add(to_code) statuses_map: dict[str, dict[str, Any]] = {} + enabled_statuses_started_at = perf_counter() all_enabled_status_rows = ( db.query(Status, StatusGroup) .outerjoin(StatusGroup, StatusGroup.id == Status.status_group_id) .filter(Status.enabled.is_(True)) .all() ) + enabled_statuses_ms = (perf_counter() - enabled_statuses_started_at) * 1000.0 for status_row, _group_row in all_enabled_status_rows: code = str(status_row.code or "").strip() if code: known_codes.add(code) if known_codes: + statuses_meta_started_at = perf_counter() status_rows = ( db.query(Status, StatusGroup) .outerjoin(StatusGroup, StatusGroup.id == Status.status_group_id) .filter(Status.code.in_(list(known_codes))) .all() ) + statuses_meta_ms = (perf_counter() - statuses_meta_started_at) * 1000.0 statuses_map = { str(status_row.code): { "name": str(status_row.name or status_row.code), @@ -271,7 +282,10 @@ def get_request_status_route_service( } for status_row, group_row in status_rows } + else: + statuses_meta_ms = 0.0 + transitions_started_at = perf_counter() transition_rows = ( db.query(TopicStatusTransition) .filter( @@ -283,6 +297,7 @@ def get_request_status_route_service( if topic_code else [] ) + transitions_ms = (perf_counter() - transitions_started_at) * 1000.0 transition_sla_by_edge: dict[tuple[str, str], int] = {} outgoing_by_status: dict[str, list[str]] = {} incoming_sla_by_status: dict[str, int] = {} @@ -479,7 +494,7 @@ def get_request_status_route_service( } ) - return { + payload = { "request_id": str(req.id), "track_number": req.track_number, "topic_code": req.topic_code, @@ -489,3 +504,19 @@ def get_request_status_route_service( "history": list(reversed(history_entries)), "nodes": nodes, } + total_ms = (perf_counter() - started_at) * 1000.0 + _STATUS_ROUTE_LOG.info( + "status_route request_id=%s total_ms=%.2f history_ms=%.2f enabled_statuses_ms=%.2f statuses_meta_ms=%.2f " + "transitions_ms=%.2f history_rows=%s known_codes=%s transition_rows=%s nodes=%s", + str(req.id), + total_ms, + history_ms, + enabled_statuses_ms, + statuses_meta_ms, + transitions_ms, + len(history_rows), + len(known_codes), + len(transition_rows), + len(nodes), + ) + return payload diff --git a/app/api/public/chat.py b/app/api/public/chat.py index ac37390..502695e 100644 --- a/app/api/public/chat.py +++ b/app/api/public/chat.py @@ -24,7 +24,8 @@ from app.services.chat_secure_service import ( list_messages_for_request, mark_messages_delivered_for_client, mark_messages_read_for_client, - serialize_message, + serialize_message_for_request, + serialize_message_bodies_for_request, serialize_messages_for_request, ) from app.services.request_read_markers import EVENT_REQUEST_DATA, mark_unread_for_lawyer @@ -175,6 +176,27 @@ def _ensure_view_access_or_403(session: dict, req: Request) -> None: raise HTTPException(status_code=404, detail="Заявка не найдена") +def _normalize_message_ids(raw: object, *, field_name: str = "ids", limit: int = 200) -> list[UUID]: + if not isinstance(raw, list): + raise HTTPException(status_code=400, detail=f'Поле "{field_name}" должно быть списком') + seen: set[UUID] = set() + out: list[UUID] = [] + for item in raw: + try: + value = UUID(str(item or "")) + except ValueError: + raise HTTPException(status_code=400, detail=f'Некорректное поле "{field_name}"') + if value in seen: + continue + seen.add(value) + out.append(value) + if len(out) >= limit: + break + if not out: + raise HTTPException(status_code=400, detail="Нужно передать хотя бы один идентификатор сообщения") + return out + + @router.get("/requests/{track_number}/messages") def list_messages_by_track( track_number: str, @@ -186,7 +208,13 @@ def list_messages_by_track( _ensure_view_access_or_403(session, req) mark_messages_read_for_client(db, request_id=req.id) rows = list_messages_for_request(db, req.id) - payload = serialize_messages_for_request(db, req.id, rows) + payload = serialize_messages_for_request( + db, + req.id, + rows, + request_extra_fields=req.extra_fields, + include_bodies=True, + ) _audit_public_chat_read( db, session=session, @@ -202,25 +230,36 @@ def list_messages_by_track( def list_messages_window_by_track( track_number: str, http_request: FastapiRequest, + before_id: str | None = None, + before_created_at: str | None = None, before_count: int = 0, limit: int = DEFAULT_CHAT_WINDOW_LIMIT, + include_body: bool = True, db: Session = Depends(get_db), session: dict = Depends(get_public_session), ): req = _request_for_track_or_404(db, track_number) _ensure_view_access_or_403(session, req) mark_messages_read_for_client(db, request_id=req.id) - rows, total, has_more, loaded_count = list_messages_for_request_window( + rows, has_more = list_messages_for_request_window( db, req.id, limit=limit, + before_id=before_id, + before_created_at=before_created_at, before_count=before_count, ) + message_total = int(get_chat_activity_summary(db, req.id).get("message_count") or len(rows)) payload = { - "rows": serialize_messages_for_request(db, req.id, rows), - "total": total, + "rows": serialize_messages_for_request( + db, + req.id, + rows, + request_extra_fields=req.extra_fields, + include_bodies=bool(include_body), + ), "has_more": has_more, - "loaded_count": loaded_count, + "total": message_total, "limit": clamp_chat_window_limit(limit), } _audit_public_chat_read( @@ -234,6 +273,44 @@ def list_messages_window_by_track( return payload +@router.post("/requests/{track_number}/message-bodies") +def load_message_bodies_by_track( + track_number: str, + payload: dict, + http_request: FastapiRequest, + db: Session = Depends(get_db), + session: dict = Depends(get_public_session), +): + req = _request_for_track_or_404(db, track_number) + _ensure_view_access_or_403(session, req) + message_ids = _normalize_message_ids((payload or {}).get("ids")) + rows = ( + db.query(Message) + .filter(Message.request_id == req.id, Message.id.in_(message_ids)) + .order_by(Message.created_at.asc(), Message.id.asc()) + .all() + ) + rows_by_id = {str(row.id): row for row in rows} + ordered_rows = [rows_by_id[str(message_id)] for message_id in message_ids if str(message_id) in rows_by_id] + result = { + "rows": serialize_message_bodies_for_request( + db, + req.id, + ordered_rows, + request_extra_fields=req.extra_fields, + ) + } + _audit_public_chat_read( + db, + session=session, + http_request=http_request, + req=req, + action="READ_CHAT_MESSAGES", + details={"rows": len(ordered_rows), "body_batch": True}, + ) + return result + + @router.post("/requests/{track_number}/messages", status_code=201) def create_message_by_track( track_number: str, @@ -246,7 +323,7 @@ def create_message_by_track( req = _request_for_track_or_404(db, track_number) _ensure_view_access_or_403(session, req) row = create_client_message(db, request=req, body=payload.body) - return serialize_message(row) + return serialize_message_for_request(row, request_extra_fields=req.extra_fields) @router.get("/requests/{track_number}/live") @@ -286,7 +363,13 @@ def get_live_chat_state_by_track( .order_by(Attachment.created_at.asc(), Attachment.id.asc()) .all() ) - delta_messages = serialize_messages_for_request(db, req.id, message_rows) + delta_messages = serialize_messages_for_request( + db, + req.id, + message_rows, + request_extra_fields=req.extra_fields, + include_bodies=True, + ) delta_attachments = [_serialize_public_attachment(row) for row in attachment_rows] subject = _require_view_session_or_403(session) @@ -492,6 +575,12 @@ def save_data_request_values( db.rollback() messages = list_messages_for_request(db, req.id) - serialized = serialize_messages_for_request(db, req.id, messages) + serialized = serialize_messages_for_request( + db, + req.id, + messages, + request_extra_fields=req.extra_fields, + include_bodies=True, + ) current = next((item for item in serialized if str(item.get("id")) == str(message_uuid)), None) return {"updated": updated, "message": current} diff --git a/app/core/http_hardening.py b/app/core/http_hardening.py index 5c35012..1c4072c 100644 --- a/app/core/http_hardening.py +++ b/app/core/http_hardening.py @@ -67,6 +67,7 @@ _PERF_PATH_PATTERNS = ( ("admin_request_detail", re.compile(r"^/api/admin/crud/requests/[^/]+$")), ("admin_chat_messages", re.compile(r"^/api/admin/chat/requests/[^/]+/messages$")), ("admin_chat_messages_window", re.compile(r"^/api/admin/chat/requests/[^/]+/messages-window$")), + ("admin_chat_message_bodies", re.compile(r"^/api/admin/chat/requests/[^/]+/message-bodies$")), ("admin_chat_live", re.compile(r"^/api/admin/chat/requests/[^/]+/live$")), ("admin_request_status_route", re.compile(r"^/api/admin/requests/[^/]+/status-route$")), ("admin_request_attachments_query", re.compile(r"^/api/admin/uploads/request-attachments/[^/]+$")), @@ -76,6 +77,7 @@ _PERF_PATH_PATTERNS = ( ("public_request_detail", re.compile(r"^/api/public/requests/[^/]+$")), ("public_chat_messages", re.compile(r"^/api/public/chat/requests/[^/]+/messages$")), ("public_chat_messages_window", re.compile(r"^/api/public/chat/requests/[^/]+/messages-window$")), + ("public_chat_message_bodies", re.compile(r"^/api/public/chat/requests/[^/]+/message-bodies$")), ("public_chat_live", re.compile(r"^/api/public/chat/requests/[^/]+/live$")), ("public_request_attachments", re.compile(r"^/api/public/requests/[^/]+/attachments$")), ("public_request_invoices", re.compile(r"^/api/public/requests/[^/]+/invoices$")), diff --git a/app/models/message.py b/app/models/message.py index 8bc2a6f..99d646d 100644 --- a/app/models/message.py +++ b/app/models/message.py @@ -1,12 +1,13 @@ import uuid from datetime import datetime -from sqlalchemy import String, Boolean, DateTime, Index -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy import String, Boolean, DateTime, Index, Text, event +from sqlalchemy.orm import Mapped, Session as OrmSession, mapped_column from sqlalchemy.dialects.postgresql import UUID from app.db.session import Base -from app.db.encrypted_types import EncryptedChatText from app.models.common import UUIDMixin, TimestampMixin +from app.models.request import Request +from app.services.chat_crypto import encrypt_message_body, encrypt_message_body_for_request, is_encrypted_message class Message(Base, UUIDMixin, TimestampMixin): __tablename__ = "messages" @@ -16,9 +17,45 @@ class Message(Base, UUIDMixin, TimestampMixin): request_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), index=True, nullable=False) author_type: Mapped[str] = mapped_column(String(20), nullable=False) # CLIENT|LAWYER|SYSTEM author_name: Mapped[str | None] = mapped_column(String(200), nullable=True) - body: Mapped[str | None] = mapped_column(EncryptedChatText(), nullable=True) + body: Mapped[str | None] = mapped_column(Text, nullable=True) immutable: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) delivered_to_client_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) delivered_to_staff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) read_by_client_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) read_by_staff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +def _find_request_for_message(session: OrmSession, request_id: uuid.UUID | None) -> Request | None: + if request_id is None: + return None + for obj in session.new: + if isinstance(obj, Request) and obj.id == request_id: + return obj + for obj in session.identity_map.values(): + if isinstance(obj, Request) and obj.id == request_id: + return obj + return session.get(Request, request_id) + + +@event.listens_for(OrmSession, "before_flush") +def _encrypt_message_bodies_before_flush(session: OrmSession, flush_context, instances) -> None: + candidates = [obj for obj in session.new if isinstance(obj, Message)] + candidates.extend(obj for obj in session.dirty if isinstance(obj, Message)) + for message in candidates: + raw_body = message.body + if raw_body is None: + continue + text = str(raw_body) + if not text or is_encrypted_message(text): + continue + request_row = _find_request_for_message(session, getattr(message, "request_id", None)) + if request_row is None: + message.body = encrypt_message_body(text) + continue + encrypted_body, next_extra_fields, changed = encrypt_message_body_for_request( + text, + request_extra_fields=request_row.extra_fields, + ) + message.body = encrypted_body + if changed: + request_row.extra_fields = next_extra_fields diff --git a/app/scripts/reencrypt_with_active_kid.py b/app/scripts/reencrypt_with_active_kid.py index 6c72e6c..05c379f 100644 --- a/app/scripts/reencrypt_with_active_kid.py +++ b/app/scripts/reencrypt_with_active_kid.py @@ -2,13 +2,15 @@ from __future__ import annotations import argparse from datetime import datetime, timezone +from uuid import UUID from sqlalchemy import text from app.db.session import SessionLocal from app.models.admin_user import AdminUser from app.models.invoice import Invoice -from app.services.chat_crypto import active_chat_kid, decrypt_message_body, encrypt_message_body, extract_message_kid, is_encrypted_message +from app.models.request import Request +from app.services.chat_crypto import decrypt_message_body, encrypt_message_body from app.services.invoice_crypto import active_requisites_kid, decrypt_requisites, encrypt_requisites, extract_requisites_kid @@ -28,8 +30,6 @@ def reencrypt_with_active_kid(*, dry_run: bool = True) -> dict[str, int]: "errors": 0, } current_data_kid = active_requisites_kid() - current_chat_kid = active_chat_kid() - try: invoice_rows = db.query(Invoice).all() counts["invoices_total"] = len(invoice_rows) @@ -63,17 +63,39 @@ def reencrypt_with_active_kid(*, dry_run: bool = True) -> dict[str, int]: except Exception: counts["errors"] += 1 - message_rows = db.execute(text("SELECT id, body FROM messages")).all() + message_rows = db.execute(text("SELECT id, request_id, body FROM messages")).all() counts["messages_total"] = len(message_rows) - for message_id, body in message_rows: + for message_id, request_id, body in message_rows: raw_body = str(body or "") if not raw_body: continue - if extract_message_kid(raw_body) == current_chat_kid: + if raw_body.startswith("chatenc:v3:"): continue try: - plaintext = decrypt_message_body(raw_body) - updated = encrypt_message_body(plaintext) + request_key = None + if request_id: + try: + request_key = UUID(str(request_id)) + except (TypeError, ValueError): + request_key = None + request_row = db.get(Request, request_key) if request_key else None + if request_row is None: + plaintext = decrypt_message_body(raw_body) + updated = encrypt_message_body(plaintext) + else: + from app.services.chat_crypto import decrypt_message_body_for_request, encrypt_message_body_for_request + + plaintext = ( + decrypt_message_body_for_request(raw_body, request_extra_fields=request_row.extra_fields) + if raw_body.startswith("chatenc:v3:") + else decrypt_message_body(raw_body) + ) + updated, next_extra_fields, changed = encrypt_message_body_for_request( + plaintext, + request_extra_fields=request_row.extra_fields, + ) + if changed: + request_row.extra_fields = next_extra_fields if updated == raw_body: continue db.execute( diff --git a/app/services/chat_crypto.py b/app/services/chat_crypto.py index cfbbee1..0d2c934 100644 --- a/app/services/chat_crypto.py +++ b/app/services/chat_crypto.py @@ -4,32 +4,107 @@ import base64 import hashlib import hmac import secrets +from typing import Any + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM from app.services.crypto_keyring import get_chat_secrets, key_digest, ordered_unique_key_digests _VERSION_LEGACY = b"v1" _PREFIX_LEGACY = "chatenc:v1:" _PREFIX_V2 = "chatenc:v2:" +_PREFIX_V3 = "chatenc:v3:" +_CHAT_CRYPTO_EXTRA_FIELDS_KEY = "chat_crypto" def _xor_bytes(a: bytes, b: bytes) -> bytes: return bytes(x ^ y for x, y in zip(a, b)) +def _urlsafe_b64encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode("ascii") + + +def _urlsafe_b64decode(value: str) -> bytes: + return base64.urlsafe_b64decode(str(value or "").encode("ascii")) + + def _aad_v2(kid: str) -> bytes: return b"v2|" + str(kid).encode("utf-8") + b"|" +def _aad_v3_message(kid: str) -> bytes: + return b"v3|message|" + str(kid).encode("utf-8") + b"|" + + +def _aad_v3_wrapped_key(kid: str) -> bytes: + return b"v3|chat-key|" + str(kid).encode("utf-8") + b"|" + + def active_chat_kid() -> str: active_kid, _ = get_chat_secrets() return active_kid +def _active_chat_secret() -> tuple[str, str]: + active_kid, key_map = get_chat_secrets() + active_secret = key_map.get(active_kid) + if not active_secret and key_map: + active_secret = next(iter(key_map.values())) + if not active_secret: + raise ValueError("Не найден активный ключ шифрования чата") + return active_kid, active_secret + + +def _chat_payload_or_none(extra_fields: dict[str, Any] | None) -> dict[str, Any] | None: + payload = (extra_fields or {}).get(_CHAT_CRYPTO_EXTRA_FIELDS_KEY) + return payload if isinstance(payload, dict) else None + + +def _wrap_chat_key(chat_key: bytes, *, kid: str, secret: str) -> dict[str, Any]: + nonce = secrets.token_bytes(12) + payload = AESGCM(key_digest(secret)).encrypt(nonce, chat_key, _aad_v3_wrapped_key(kid)) + return { + "version": 1, + "kek_kid": str(kid), + "nonce": _urlsafe_b64encode(nonce), + "wrapped_key": _urlsafe_b64encode(payload), + } + + +def _unwrap_chat_key(payload: dict[str, Any], *, key_map: dict[str, str]) -> tuple[bytes, str]: + if int(payload.get("version") or 0) != 1: + raise ValueError("Неподдерживаемая версия ключа чата") + kid = str(payload.get("kek_kid") or "").strip() + nonce = _urlsafe_b64decode(str(payload.get("nonce") or "")) + wrapped_key = _urlsafe_b64decode(str(payload.get("wrapped_key") or "")) + if len(nonce) != 12 or not wrapped_key: + raise ValueError("Некорректный формат ключа чата") + + candidate_secrets: list[tuple[str, str]] = [] + if kid and kid in key_map: + candidate_secrets.append((kid, key_map[kid])) + for fallback_kid, secret in key_map.items(): + if kid and fallback_kid == kid: + continue + candidate_secrets.append((fallback_kid, secret)) + + for candidate_kid, secret in candidate_secrets: + try: + plaintext = AESGCM(key_digest(secret)).decrypt(nonce, wrapped_key, _aad_v3_wrapped_key(kid or candidate_kid)) + except Exception: + continue + if len(plaintext) not in {16, 24, 32}: + raise ValueError("Некорректная длина ключа чата") + return plaintext, (kid or candidate_kid) + raise ValueError("Не удалось расшифровать ключ чата") + + def extract_message_kid(value: str | None) -> str | None: token = str(value or "").strip() if not token: return None - if token.startswith(_PREFIX_V2): + if token.startswith(_PREFIX_V2) or token.startswith(_PREFIX_V3): parts = token.split(":", 3) if len(parts) != 4: return None @@ -40,22 +115,54 @@ def extract_message_kid(value: str | None) -> str | None: def is_encrypted_message(value: str | None) -> bool: token = str(value or "").strip() - return token.startswith(_PREFIX_LEGACY) or token.startswith(_PREFIX_V2) + return token.startswith(_PREFIX_LEGACY) or token.startswith(_PREFIX_V2) or token.startswith(_PREFIX_V3) + + +def prepare_request_chat_crypto(extra_fields: dict[str, Any] | None) -> tuple[dict[str, Any], bytes, bool]: + active_kid, key_map = get_chat_secrets() + updated = dict(extra_fields or {}) + payload = _chat_payload_or_none(updated) + chat_key: bytes | None = None + payload_kid = active_kid + changed = False + + if payload: + try: + chat_key, payload_kid = _unwrap_chat_key(payload, key_map=key_map) + except Exception: + chat_key = None + + if chat_key is None: + chat_key = secrets.token_bytes(32) + changed = True + + if changed or payload_kid != active_kid or payload != _chat_payload_or_none(updated): + active_secret = key_map.get(active_kid) + if not active_secret: + raise ValueError("Не найден активный ключ шифрования чата") + updated[_CHAT_CRYPTO_EXTRA_FIELDS_KEY] = _wrap_chat_key(chat_key, kid=active_kid, secret=active_secret) + changed = True + + return updated, chat_key, changed + + +def _request_chat_key(extra_fields: dict[str, Any] | None) -> tuple[bytes, str]: + payload = _chat_payload_or_none(extra_fields) + if not payload: + raise ValueError("Не найден ключ шифрования чата для заявки") + key_map = get_chat_secrets()[1] + chat_key, payload_kid = _unwrap_chat_key(payload, key_map=key_map) + return chat_key, payload_kid def encrypt_message_body(value: str | None) -> str | None: if value is None: return None text = str(value) - if not text: - return text - if is_encrypted_message(text): + if not text or is_encrypted_message(text): return text - active_kid, key_map = get_chat_secrets() - active_secret = key_map.get(active_kid) - if not active_secret: - raise ValueError("Не найден активный ключ шифрования чата") + active_kid, active_secret = _active_chat_secret() key = key_digest(active_secret) raw = text.encode("utf-8") @@ -64,11 +171,37 @@ def encrypt_message_body(value: str | None) -> str | None: cipher = _xor_bytes(raw, stream) tag = hmac.new(key, _aad_v2(active_kid) + nonce + cipher, hashlib.sha256).digest() blob = nonce + tag + cipher - return f"{_PREFIX_V2}{active_kid}:" + base64.urlsafe_b64encode(blob).decode("ascii") + return f"{_PREFIX_V2}{active_kid}:" + _urlsafe_b64encode(blob) + + +def encrypt_message_body_for_request( + value: str | None, + *, + request_extra_fields: dict[str, Any] | None, +) -> tuple[str | None, dict[str, Any], bool]: + if value is None: + return None, dict(request_extra_fields or {}), False + text = str(value) + if not text or is_encrypted_message(text): + return text, dict(request_extra_fields or {}), False + + updated_extra_fields, chat_key, changed = prepare_request_chat_crypto(request_extra_fields) + kid = str(extract_request_chat_kek_kid(updated_extra_fields) or active_chat_kid()) + nonce = secrets.token_bytes(12) + cipher = AESGCM(chat_key).encrypt(nonce, text.encode("utf-8"), _aad_v3_message(kid)) + return f"{_PREFIX_V3}{kid}:" + _urlsafe_b64encode(nonce + cipher), updated_extra_fields, changed + + +def extract_request_chat_kek_kid(extra_fields: dict[str, Any] | None) -> str | None: + payload = _chat_payload_or_none(extra_fields) + if not payload: + return None + kid = str(payload.get("kek_kid") or "").strip() + return kid or None def _decrypt_v2(encoded: str, *, kid: str, key: bytes) -> str: - blob = base64.urlsafe_b64decode(encoded.encode("ascii")) + blob = _urlsafe_b64decode(encoded) if len(blob) < 16 + 32: raise ValueError("Некорректный зашифрованный формат сообщения") nonce = blob[:16] @@ -82,8 +215,19 @@ def _decrypt_v2(encoded: str, *, kid: str, key: bytes) -> str: return raw.decode("utf-8") +def _decrypt_v3(encoded: str, *, kid: str, request_extra_fields: dict[str, Any] | None) -> str: + chat_key, _ = _request_chat_key(request_extra_fields) + blob = _urlsafe_b64decode(encoded) + if len(blob) <= 12: + raise ValueError("Некорректный зашифрованный формат сообщения") + nonce = blob[:12] + cipher = blob[12:] + raw = AESGCM(chat_key).decrypt(nonce, cipher, _aad_v3_message(kid)) + return raw.decode("utf-8") + + def _decrypt_legacy(encoded: str, keys: list[bytes]) -> str: - blob = base64.urlsafe_b64decode(encoded.encode("ascii")) + blob = _urlsafe_b64decode(encoded) if len(blob) < 2 + 16 + 32: raise ValueError("Некорректный зашифрованный формат сообщения") version = blob[:2] @@ -115,6 +259,8 @@ def decrypt_message_body(value: str | None) -> str | None: active_kid, key_map = get_chat_secrets() _ = active_kid + if text.startswith(_PREFIX_V3): + raise ValueError("Для сообщений v3 требуется контекст заявки") if text.startswith(_PREFIX_V2): encoded = text[len(_PREFIX_V2) :] parts = encoded.split(":", 1) @@ -132,3 +278,23 @@ def decrypt_message_body(value: str | None) -> str | None: encoded = text[len(_PREFIX_LEGACY) :] return _decrypt_legacy(encoded, ordered_unique_key_digests(key_map.values())) + + +def decrypt_message_body_for_request( + value: str | None, + *, + request_extra_fields: dict[str, Any] | None, +) -> str | None: + if value is None: + return None + text = str(value) + if not text or not is_encrypted_message(text): + return text + if text.startswith(_PREFIX_V3): + encoded = text[len(_PREFIX_V3) :] + parts = encoded.split(":", 1) + if len(parts) != 2: + raise ValueError("Некорректный зашифрованный формат сообщения") + kid, payload = str(parts[0] or "").strip(), parts[1] + return _decrypt_v3(payload, kid=kid, request_extra_fields=request_extra_fields) + return decrypt_message_body(text) diff --git a/app/services/chat_secure_service.py b/app/services/chat_secure_service.py index 66fe167..b7ffdba 100644 --- a/app/services/chat_secure_service.py +++ b/app/services/chat_secure_service.py @@ -1,24 +1,29 @@ from __future__ import annotations +import logging +from time import perf_counter import uuid from datetime import datetime, timezone from typing import Any from fastapi import HTTPException -from sqlalchemy import func +from sqlalchemy import and_, func, or_ from sqlalchemy.orm import Session from app.models.attachment import Attachment from app.models.message import Message from app.models.request import Request from app.models.request_data_requirement import RequestDataRequirement +from app.services.chat_crypto import decrypt_message_body_for_request from app.services.notifications import EVENT_MESSAGE as NOTIFICATION_EVENT_MESSAGE, notify_request_event from app.services.request_read_markers import EVENT_MESSAGE, mark_unread_for_client, mark_unread_for_lawyer MAX_CHAT_MESSAGE_LEN = 12_000 DEFAULT_CHAT_WINDOW_LIMIT = 50 MAX_CHAT_WINDOW_LIMIT = 200 +MAX_CHAT_BODY_BATCH = 200 CHAT_PARTICIPANT_ADMIN_IDS_KEY = "chat_participant_admin_ids" +_CHAT_WORKSPACE_LOG = logging.getLogger("uvicorn.error") def _normalize_message_body(body: str | None) -> str: @@ -49,19 +54,75 @@ def clamp_chat_window_limit(limit: int | None) -> int: return max(1, min(normalized, MAX_CHAT_WINDOW_LIMIT)) +def clamp_chat_body_batch_limit(limit: int | None) -> int: + if limit is None: + return MAX_CHAT_BODY_BATCH + try: + normalized = int(limit) + except (TypeError, ValueError): + normalized = MAX_CHAT_BODY_BATCH + return max(1, min(normalized, MAX_CHAT_BODY_BATCH)) + + +def _parse_window_message_uuid(raw: str | None) -> uuid.UUID | None: + value = str(raw or "").strip() + if not value: + return None + try: + return uuid.UUID(value) + except (TypeError, ValueError): + return None + + +def _parse_window_datetime(raw: str | None) -> datetime | None: + value = str(raw or "").strip() + if not value: + return None + normalized = value.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + def list_messages_for_request_window( db: Session, request_id: Any, *, limit: int | None, + before_id: str | None = None, + before_created_at: str | None = None, before_count: int = 0, -) -> tuple[list[Message], int, bool, int]: +) -> tuple[list[Message], bool]: window_limit = clamp_chat_window_limit(limit) - loaded_count = max(0, int(before_count or 0)) base_query = db.query(Message).filter(Message.request_id == request_id) + before_uuid = _parse_window_message_uuid(before_id) + before_dt = _parse_window_datetime(before_created_at) + + if before_uuid is not None and before_dt is not None: + base_query = base_query.filter( + or_( + Message.created_at < before_dt, + and_(Message.created_at == before_dt, Message.id < before_uuid), + ) + ) + rows_desc = ( + base_query + .order_by(Message.created_at.desc(), Message.id.desc()) + .limit(window_limit + 1) + .all() + ) + has_more = len(rows_desc) > window_limit + rows = list(reversed(rows_desc[:window_limit])) + return rows, has_more + + loaded_count = max(0, int(before_count or 0)) total = int(base_query.count() or 0) if total <= 0 or loaded_count >= total: - return [], total, False, loaded_count + return [], False remaining = total - loaded_count window_size = min(window_limit, remaining) @@ -73,9 +134,8 @@ def list_messages_for_request_window( .limit(window_size) .all() ) - next_loaded_count = loaded_count + len(rows) has_more = offset > 0 - return rows, total, has_more, next_loaded_count + return rows, has_more def _iso_or_none(value: datetime | None) -> str | None: @@ -160,13 +220,14 @@ def mark_messages_read_for_staff(db: Session, *, request_id: Any, commit: bool = return _mark_counterparty_delivery(db, request_id=request_id, recipient="STAFF", mark_read=True, commit=commit) -def serialize_message(row: Message) -> dict[str, Any]: +def serialize_message(row: Message, *, body: str | None = None, body_loaded: bool = True) -> dict[str, Any]: return { "id": str(row.id), "request_id": str(row.request_id), "author_type": row.author_type, "author_name": row.author_name, - "body": row.body, + "body": body, + "body_loaded": bool(body_loaded), "message_kind": "TEXT", "request_data_items": [], "request_data_all_filled": False, @@ -186,6 +247,30 @@ def _truncate_request_data_label(label: str, limit: int = 18) -> str: return text[: max(3, limit - 3)].rstrip() + "..." +def _message_uuid_list(rows: list[Message]) -> list[uuid.UUID]: + out: list[uuid.UUID] = [] + for row in rows: + message_id = getattr(row, "id", None) + if isinstance(message_id, uuid.UUID): + out.append(message_id) + return out + + +def _request_data_message_ids(db: Session, request_id: Any, message_ids: list[uuid.UUID]) -> set[str]: + if not message_ids: + return set() + rows = ( + db.query(RequestDataRequirement.request_message_id) + .filter( + RequestDataRequirement.request_id == request_id, + RequestDataRequirement.request_message_id.in_(message_ids), + ) + .distinct() + .all() + ) + return {str(item[0]) for item in rows if item and item[0] is not None} + + def _normalize_admin_uuid(value: str | None) -> str | None: raw = str(value or "").strip() if not raw: @@ -218,13 +303,17 @@ def _register_chat_participant(request: Request, admin_user_id: str | None) -> N request.extra_fields = extra -def serialize_messages_for_request(db: Session, request_id: Any, rows: list[Message]) -> list[dict[str, Any]]: - message_ids = [] - for row in rows: - try: - message_ids.append(row.id) - except Exception: - continue +def serialize_messages_for_request( + db: Session, + request_id: Any, + rows: list[Message], + *, + request_extra_fields: dict[str, Any] | None = None, + include_bodies: bool = True, +) -> list[dict[str, Any]]: + started_at = perf_counter() + message_ids = _message_uuid_list(rows) + requirements_started_at = perf_counter() requirements = ( db.query(RequestDataRequirement) .filter( @@ -241,6 +330,7 @@ def serialize_messages_for_request(db: Session, request_id: Any, rows: list[Mess if message_ids else [] ) + requirements_ms = (perf_counter() - requirements_started_at) * 1000.0 by_message_id: dict[str, list[RequestDataRequirement]] = {} for item in requirements: mid = str(item.request_message_id or "").strip() @@ -260,13 +350,28 @@ def serialize_messages_for_request(db: Session, request_id: Any, rows: list[Mess continue attachment_map: dict[str, Attachment] = {} if file_attachment_ids: + attachment_lookup_started_at = perf_counter() attachment_rows = db.query(Attachment).filter(Attachment.id.in_(file_attachment_ids)).all() attachment_map = {str(row.id): row for row in attachment_rows} + attachment_lookup_ms = (perf_counter() - attachment_lookup_started_at) * 1000.0 + else: + attachment_lookup_ms = 0.0 out: list[dict[str, Any]] = [] for row in rows: - payload = serialize_message(row) linked = by_message_id.get(str(row.id), []) + is_request_data = bool(linked) + if is_request_data: + body_value = "Запрос" + body_loaded = True + elif include_bodies: + body_value = decrypt_message_body_for_request(row.body, request_extra_fields=request_extra_fields) + body_loaded = True + else: + body_value = None + body_loaded = False + + payload = serialize_message(row, body=body_value, body_loaded=body_loaded) if linked: linked_sorted = sorted( linked, @@ -315,9 +420,56 @@ def serialize_messages_for_request(db: Session, request_id: Any, rows: list[Mess else: payload["message_kind"] = "TEXT" out.append(payload) + total_ms = (perf_counter() - started_at) * 1000.0 + _CHAT_WORKSPACE_LOG.info( + "serialize_messages request_id=%s total_ms=%.2f requirements_ms=%.2f attachment_lookup_ms=%.2f rows=%s requirements=%s file_requirements=%s", + str(request_id), + total_ms, + requirements_ms, + attachment_lookup_ms, + len(rows), + len(requirements), + len(file_attachment_ids), + ) return out +def serialize_message_bodies_for_request( + db: Session, + request_id: Any, + rows: list[Message], + *, + request_extra_fields: dict[str, Any] | None, +) -> list[dict[str, Any]]: + request_data_ids = _request_data_message_ids(db, request_id, _message_uuid_list(rows)) + payload: list[dict[str, Any]] = [] + for row in rows: + row_id = str(row.id) + if row_id in request_data_ids: + payload.append({"id": row_id, "body": "Запрос", "body_loaded": True}) + continue + payload.append( + { + "id": row_id, + "body": decrypt_message_body_for_request(row.body, request_extra_fields=request_extra_fields), + "body_loaded": True, + } + ) + return payload + + +def serialize_message_for_request( + row: Message, + *, + request_extra_fields: dict[str, Any] | None, +) -> dict[str, Any]: + return serialize_message( + row, + body=decrypt_message_body_for_request(row.body, request_extra_fields=request_extra_fields), + body_loaded=True, + ) + + def create_client_message( db: Session, *, diff --git a/app/web/admin.js b/app/web/admin.js index fc2193c..5879d53 100644 --- a/app/web/admin.js +++ b/app/web/admin.js @@ -5124,7 +5124,7 @@ } } : void 0 }, - String(entry.payload?.message_kind || "") === "REQUEST_DATA" ? /* @__PURE__ */ React.createElement(React.Fragment, null, /* @__PURE__ */ React.createElement("div", { className: "chat-request-data-head" }, "\u0417\u0430\u043F\u0440\u043E\u0441"), renderRequestDataMessageItems(entry.payload)) : /* @__PURE__ */ React.createElement(React.Fragment, null, serviceMessageContent?.title ? /* @__PURE__ */ React.createElement("div", { className: "chat-service-head" }, serviceMessageContent.title) : null, serviceMessageContent ? serviceMessageContent.text ? /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, serviceMessageContent.text) : null : /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, String(entry.payload?.body || ""))), + String(entry.payload?.message_kind || "") === "REQUEST_DATA" ? /* @__PURE__ */ React.createElement(React.Fragment, null, /* @__PURE__ */ React.createElement("div", { className: "chat-request-data-head" }, "\u0417\u0430\u043F\u0440\u043E\u0441"), renderRequestDataMessageItems(entry.payload)) : /* @__PURE__ */ React.createElement(React.Fragment, null, serviceMessageContent?.title ? /* @__PURE__ */ React.createElement("div", { className: "chat-service-head" }, serviceMessageContent.title) : null, serviceMessageContent ? serviceMessageContent.text ? /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, serviceMessageContent.text) : null : /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, entry.payload?.body_loaded === false ? "\u0417\u0430\u0433\u0440\u0443\u0437\u043A\u0430 \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u044F..." : String(entry.payload?.body || ""))), (() => { if (String(entry.payload?.message_kind || "") === "REQUEST_DATA") return null; const messageId = String(entry.payload?.id || "").trim(); @@ -6131,6 +6131,18 @@ }); return sortRowsByCreatedAt(Array.from(merged.values())); } + function getOldestMessageCursor(rows) { + const sorted = sortRowsByCreatedAt(Array.isArray(rows) ? rows : []); + const first = sorted[0]; + if (!first) return null; + const beforeId = String(first.id || "").trim(); + const beforeCreatedAt = String(first.created_at || first.updated_at || "").trim(); + if (!beforeId || !beforeCreatedAt) return null; + return { beforeId, beforeCreatedAt }; + } + function collectDeferredMessageIds(rows) { + return (Array.isArray(rows) ? rows : []).filter((row) => row && typeof row === "object" && row.body_loaded === false && String(row.id || "").trim()).map((row) => String(row.id).trim()); + } function normalizeMessageAuthors(rows, users) { const usersByEmail = new Map( (Array.isArray(users) ? users : []).filter((user) => user && user.email).map((user) => [String(user.email).toLowerCase(), String(user.name || user.email)]) @@ -6146,6 +6158,30 @@ return item; }); } + function buildFinanceSummaryFromInvoices(financeSummaryData, rowData, invoices) { + if (financeSummaryData && typeof financeSummaryData === "object") return financeSummaryData; + const paidInvoices = (Array.isArray(invoices) ? invoices : []).filter( + (item) => String(item?.status || "").toUpperCase() === "PAID" + ); + const paidTotal = paidInvoices.reduce((acc, item) => { + const amount = Number(item?.amount || 0); + return Number.isFinite(amount) ? acc + amount : acc; + }, 0); + const latestPaidAt = paidInvoices.reduce((latest, item) => { + const raw = item?.paid_at; + const ts = raw ? new Date(raw).getTime() : Number.NaN; + if (!Number.isFinite(ts)) return latest; + if (!latest) return String(raw); + const latestTs = new Date(latest).getTime(); + return ts > latestTs ? String(raw) : latest; + }, ""); + return { + request_cost: rowData?.request_cost ?? null, + effective_rate: rowData?.effective_rate ?? null, + paid_total: Math.round((paidTotal + Number.EPSILON) * 100) / 100, + last_paid_at: latestPaidAt || rowData?.paid_at || null + }; + } function useRequestWorkspace(options) { const { useCallback, useRef, useState } = React; const opts = options || {}; @@ -6161,6 +6197,32 @@ setRequestModal(createRequestModalState()); requestOpenGuardRef.current = { requestId: "", ts: 0 }; }, []); + const hydrateRequestMessageBodies = useCallback( + async (requestId, rows) => { + const targetRequestId = String(requestId || "").trim(); + const ids = collectDeferredMessageIds(rows); + if (!api || !targetRequestId || !ids.length) return null; + try { + const payload = await api("/api/admin/chat/requests/" + targetRequestId + "/message-bodies", { + method: "POST", + body: { ids } + }); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; + if (!nextRows.length) return payload || null; + setRequestModal((prev) => { + if (String(prev.requestId || "") !== targetRequestId) return prev; + return { + ...prev, + messages: mergeRowsById(prev.messages, nextRows) + }; + }); + return payload || null; + } catch (_) { + return null; + } + }, + [api] + ); const updateRequestModalMessageDraft = useCallback((event) => { const value = event.target.value; setRequestModal((prev) => ({ ...prev, messageDraft: value })); @@ -6275,12 +6337,10 @@ })); } try { - const workspaceData = await api("/api/admin/requests/" + requestId + "/workspace"); + const workspaceData = await api("/api/admin/requests/" + requestId + "/workspace?include_related=false"); const row = workspaceData?.request || null; const messagesData = { rows: workspaceData?.messages || [] }; - const attachmentsData = { rows: workspaceData?.attachments || [] }; const statusRouteData = workspaceData?.status_route || { nodes: [] }; - const invoicesData = { rows: workspaceData?.invoices || [] }; const financeSummaryData = workspaceData?.finance_summary || null; const usersById = new Map(users.filter((user) => user && user.id).map((user) => [String(user.id), user])); const rowData = row && typeof row === "object" ? { ...row } : row; @@ -6294,40 +6354,15 @@ } } } - const attachments = (attachmentsData.rows || []).map((item) => ({ - ...item, - download_url: resolveAdminObjectSrc2(item.s3_key, token) - })); const normalizedMessages = normalizeMessageAuthors(messagesData.rows || [], users); - const invoices = Array.isArray(invoicesData?.rows) ? invoicesData.rows : []; - const paidInvoices = invoices.filter( - (item) => String(item?.status || "").toUpperCase() === "PAID" - ); - const paidTotal = paidInvoices.reduce((acc, item) => { - const amount = Number(item?.amount || 0); - return Number.isFinite(amount) ? acc + amount : acc; - }, 0); - const latestPaidAt = paidInvoices.reduce((latest, item) => { - const raw = item?.paid_at; - const ts = raw ? new Date(raw).getTime() : Number.NaN; - if (!Number.isFinite(ts)) return latest; - if (!latest) return String(raw); - const latestTs = new Date(latest).getTime(); - return ts > latestTs ? String(raw) : latest; - }, ""); setRequestModal((prev) => ({ ...prev, loading: false, requestId: rowData?.id || requestId, trackNumber: String(rowData?.track_number || ""), requestData: rowData, - financeSummary: financeSummaryData || { - request_cost: rowData?.request_cost ?? null, - effective_rate: rowData?.effective_rate ?? null, - paid_total: Math.round((paidTotal + Number.EPSILON) * 100) / 100, - last_paid_at: latestPaidAt || rowData?.paid_at || null - }, - invoices, + financeSummary: buildFinanceSummaryFromInvoices(financeSummaryData, rowData, []), + invoices: [], statusRouteNodes: Array.isArray(statusRouteData?.nodes) ? statusRouteData.nodes : [], statusHistory: Array.isArray(statusRouteData?.history) ? statusRouteData.history : [], availableStatuses: Array.isArray(statusRouteData?.available_statuses) ? statusRouteData.available_statuses : [], @@ -6337,10 +6372,35 @@ messagesLoadingMore: false, messagesLoadedCount: Number(workspaceData?.messages_loaded_count || normalizedMessages.length || 0), messagesTotal: Number(workspaceData?.messages_total || normalizedMessages.length || 0), - attachments, + attachments: [], selectedFiles: [], fileUploading: false })); + void hydrateRequestMessageBodies(requestId, normalizedMessages); + void Promise.all([ + api("/api/admin/uploads/request-attachments/" + requestId), + api("/api/admin/invoices/by-request/" + requestId), + api("/api/admin/requests/" + requestId + "/status-route") + ]).then(([attachmentsData, invoicesData, nextStatusRouteData]) => { + const attachments = (attachmentsData?.rows || []).map((item) => ({ + ...item, + download_url: resolveAdminObjectSrc2(item.s3_key, token) + })); + const invoices = Array.isArray(invoicesData?.rows) ? invoicesData.rows : []; + setRequestModal((prev) => { + if (String(prev.requestId || "") !== String(requestId)) return prev; + return { + ...prev, + attachments, + invoices, + financeSummary: buildFinanceSummaryFromInvoices(prev.financeSummary, prev.requestData, invoices), + statusRouteNodes: Array.isArray(nextStatusRouteData?.nodes) ? nextStatusRouteData.nodes : [], + statusHistory: Array.isArray(nextStatusRouteData?.history) ? nextStatusRouteData.history : [], + availableStatuses: Array.isArray(nextStatusRouteData?.available_statuses) ? nextStatusRouteData.available_statuses : [], + currentImportantDateAt: String(nextStatusRouteData?.current_important_date_at || prev.currentImportantDateAt || "") + }; + }); + }).catch(() => null); if (showLoading && typeof setStatus === "function") setStatus("requestModal", "", ""); } catch (error) { setRequestModal((prev) => ({ @@ -6366,7 +6426,7 @@ if (typeof setStatus === "function") setStatus("requestModal", "\u041E\u0448\u0438\u0431\u043A\u0430: " + error.message, "error"); } }, - [api, resolveAdminObjectSrc2, setStatus, token, users] + [api, hydrateRequestMessageBodies, resolveAdminObjectSrc2, setStatus, token, users] ); const refreshRequestModal = useCallback(async () => { if (!requestModal.requestId) return; @@ -6527,22 +6587,33 @@ ); const loadOlderRequestMessages = useCallback(async () => { const requestId = String(requestModal.requestId || "").trim(); - const loadedCount = Number(requestModal.messagesLoadedCount || 0); + const cursor = getOldestMessageCursor(requestModal.messages); if (!api || !requestId || requestModal.messagesLoadingMore || !requestModal.messagesHasMore) return null; setRequestModal((prev) => ({ ...prev, messagesLoadingMore: true })); try { - const payload = await api( - "/api/admin/chat/requests/" + requestId + "/messages-window?before_count=" + encodeURIComponent(String(loadedCount)) - ); + const query = new URLSearchParams({ include_body: "false" }); + if (cursor?.beforeId && cursor?.beforeCreatedAt) { + query.set("before_id", cursor.beforeId); + query.set("before_created_at", cursor.beforeCreatedAt); + } else { + query.set("before_count", String(Number(requestModal.messagesLoadedCount || 0))); + } + const payload = await api("/api/admin/chat/requests/" + requestId + "/messages-window?" + query.toString()); const nextMessages = normalizeMessageAuthors(payload?.rows || [], users); setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false, - messages: mergeRowsById(nextMessages, prev.messages), - messagesHasMore: Boolean(payload?.has_more), - messagesLoadedCount: Number(payload?.loaded_count || prev.messagesLoadedCount || 0), - messagesTotal: Number(payload?.total || prev.messagesTotal || 0) + ...(function() { + const merged = mergeRowsById(nextMessages, prev.messages); + return { + messages: merged, + messagesHasMore: Boolean(payload?.has_more), + messagesLoadedCount: merged.length, + messagesTotal: Number(prev.messagesTotal || merged.length || 0) + }; + })() })); + void hydrateRequestMessageBodies(requestId, nextMessages); return payload || null; } catch (error) { setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false })); @@ -6556,6 +6627,7 @@ requestModal.messagesLoadingMore, requestModal.requestId, setStatus, + hydrateRequestMessageBodies, users ]); const setRequestTyping = useCallback( diff --git a/app/web/admin/features/requests/RequestWorkspace.jsx b/app/web/admin/features/requests/RequestWorkspace.jsx index ae69609..9f883e2 100644 --- a/app/web/admin/features/requests/RequestWorkspace.jsx +++ b/app/web/admin/features/requests/RequestWorkspace.jsx @@ -1761,7 +1761,9 @@ export function RequestWorkspace({ {serviceMessageContent ? ( serviceMessageContent.text ?

{serviceMessageContent.text}

: null ) : ( -

{String(entry.payload?.body || "")}

+

+ {entry.payload?.body_loaded === false ? "Загрузка сообщения..." : String(entry.payload?.body || "")} +

)} )} diff --git a/app/web/admin/hooks/useRequestWorkspace.js b/app/web/admin/hooks/useRequestWorkspace.js index 25377f5..eca2325 100644 --- a/app/web/admin/hooks/useRequestWorkspace.js +++ b/app/web/admin/hooks/useRequestWorkspace.js @@ -77,6 +77,22 @@ function mergeRowsById(existingRows, incomingRows) { return sortRowsByCreatedAt(Array.from(merged.values())); } +function getOldestMessageCursor(rows) { + const sorted = sortRowsByCreatedAt(Array.isArray(rows) ? rows : []); + const first = sorted[0]; + if (!first) return null; + const beforeId = String(first.id || "").trim(); + const beforeCreatedAt = String(first.created_at || first.updated_at || "").trim(); + if (!beforeId || !beforeCreatedAt) return null; + return { beforeId, beforeCreatedAt }; +} + +function collectDeferredMessageIds(rows) { + return (Array.isArray(rows) ? rows : []) + .filter((row) => row && typeof row === "object" && row.body_loaded === false && String(row.id || "").trim()) + .map((row) => String(row.id).trim()); +} + function normalizeMessageAuthors(rows, users) { const usersByEmail = new Map( (Array.isArray(users) ? users : []) @@ -95,6 +111,31 @@ function normalizeMessageAuthors(rows, users) { }); } +function buildFinanceSummaryFromInvoices(financeSummaryData, rowData, invoices) { + if (financeSummaryData && typeof financeSummaryData === "object") return financeSummaryData; + const paidInvoices = (Array.isArray(invoices) ? invoices : []).filter( + (item) => String(item?.status || "").toUpperCase() === "PAID" + ); + const paidTotal = paidInvoices.reduce((acc, item) => { + const amount = Number(item?.amount || 0); + return Number.isFinite(amount) ? acc + amount : acc; + }, 0); + const latestPaidAt = paidInvoices.reduce((latest, item) => { + const raw = item?.paid_at; + const ts = raw ? new Date(raw).getTime() : Number.NaN; + if (!Number.isFinite(ts)) return latest; + if (!latest) return String(raw); + const latestTs = new Date(latest).getTime(); + return ts > latestTs ? String(raw) : latest; + }, ""); + return { + request_cost: rowData?.request_cost ?? null, + effective_rate: rowData?.effective_rate ?? null, + paid_total: Math.round((paidTotal + Number.EPSILON) * 100) / 100, + last_paid_at: latestPaidAt || rowData?.paid_at || null, + }; +} + export function useRequestWorkspace(options) { const { useCallback, useRef, useState } = React; const opts = options || {}; @@ -113,6 +154,33 @@ export function useRequestWorkspace(options) { requestOpenGuardRef.current = { requestId: "", ts: 0 }; }, []); + const hydrateRequestMessageBodies = useCallback( + async (requestId, rows) => { + const targetRequestId = String(requestId || "").trim(); + const ids = collectDeferredMessageIds(rows); + if (!api || !targetRequestId || !ids.length) return null; + try { + const payload = await api("/api/admin/chat/requests/" + targetRequestId + "/message-bodies", { + method: "POST", + body: { ids }, + }); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; + if (!nextRows.length) return payload || null; + setRequestModal((prev) => { + if (String(prev.requestId || "") !== targetRequestId) return prev; + return { + ...prev, + messages: mergeRowsById(prev.messages, nextRows), + }; + }); + return payload || null; + } catch (_) { + return null; + } + }, + [api] + ); + const updateRequestModalMessageDraft = useCallback((event) => { const value = event.target.value; setRequestModal((prev) => ({ ...prev, messageDraft: value })); @@ -240,12 +308,10 @@ export function useRequestWorkspace(options) { } try { - const workspaceData = await api("/api/admin/requests/" + requestId + "/workspace"); + const workspaceData = await api("/api/admin/requests/" + requestId + "/workspace?include_related=false"); const row = workspaceData?.request || null; const messagesData = { rows: workspaceData?.messages || [] }; - const attachmentsData = { rows: workspaceData?.attachments || [] }; const statusRouteData = workspaceData?.status_route || { nodes: [] }; - const invoicesData = { rows: workspaceData?.invoices || [] }; const financeSummaryData = workspaceData?.finance_summary || null; const usersById = new Map(users.filter((user) => user && user.id).map((user) => [String(user.id), user])); const rowData = row && typeof row === "object" ? { ...row } : row; @@ -259,40 +325,15 @@ export function useRequestWorkspace(options) { } } } - const attachments = (attachmentsData.rows || []).map((item) => ({ - ...item, - download_url: resolveAdminObjectSrc(item.s3_key, token), - })); const normalizedMessages = normalizeMessageAuthors(messagesData.rows || [], users); - const invoices = Array.isArray(invoicesData?.rows) ? invoicesData.rows : []; - const paidInvoices = invoices.filter( - (item) => String(item?.status || "").toUpperCase() === "PAID" - ); - const paidTotal = paidInvoices.reduce((acc, item) => { - const amount = Number(item?.amount || 0); - return Number.isFinite(amount) ? acc + amount : acc; - }, 0); - const latestPaidAt = paidInvoices.reduce((latest, item) => { - const raw = item?.paid_at; - const ts = raw ? new Date(raw).getTime() : Number.NaN; - if (!Number.isFinite(ts)) return latest; - if (!latest) return String(raw); - const latestTs = new Date(latest).getTime(); - return ts > latestTs ? String(raw) : latest; - }, ""); setRequestModal((prev) => ({ ...prev, loading: false, requestId: rowData?.id || requestId, trackNumber: String(rowData?.track_number || ""), requestData: rowData, - financeSummary: financeSummaryData || { - request_cost: rowData?.request_cost ?? null, - effective_rate: rowData?.effective_rate ?? null, - paid_total: Math.round((paidTotal + Number.EPSILON) * 100) / 100, - last_paid_at: latestPaidAt || rowData?.paid_at || null, - }, - invoices, + financeSummary: buildFinanceSummaryFromInvoices(financeSummaryData, rowData, []), + invoices: [], statusRouteNodes: Array.isArray(statusRouteData?.nodes) ? statusRouteData.nodes : [], statusHistory: Array.isArray(statusRouteData?.history) ? statusRouteData.history : [], availableStatuses: Array.isArray(statusRouteData?.available_statuses) ? statusRouteData.available_statuses : [], @@ -302,10 +343,37 @@ export function useRequestWorkspace(options) { messagesLoadingMore: false, messagesLoadedCount: Number(workspaceData?.messages_loaded_count || normalizedMessages.length || 0), messagesTotal: Number(workspaceData?.messages_total || normalizedMessages.length || 0), - attachments, + attachments: [], selectedFiles: [], fileUploading: false, })); + void hydrateRequestMessageBodies(requestId, normalizedMessages); + void Promise.all([ + api("/api/admin/uploads/request-attachments/" + requestId), + api("/api/admin/invoices/by-request/" + requestId), + api("/api/admin/requests/" + requestId + "/status-route"), + ]) + .then(([attachmentsData, invoicesData, nextStatusRouteData]) => { + const attachments = (attachmentsData?.rows || []).map((item) => ({ + ...item, + download_url: resolveAdminObjectSrc(item.s3_key, token), + })); + const invoices = Array.isArray(invoicesData?.rows) ? invoicesData.rows : []; + setRequestModal((prev) => { + if (String(prev.requestId || "") !== String(requestId)) return prev; + return { + ...prev, + attachments, + invoices, + financeSummary: buildFinanceSummaryFromInvoices(prev.financeSummary, prev.requestData, invoices), + statusRouteNodes: Array.isArray(nextStatusRouteData?.nodes) ? nextStatusRouteData.nodes : [], + statusHistory: Array.isArray(nextStatusRouteData?.history) ? nextStatusRouteData.history : [], + availableStatuses: Array.isArray(nextStatusRouteData?.available_statuses) ? nextStatusRouteData.available_statuses : [], + currentImportantDateAt: String(nextStatusRouteData?.current_important_date_at || prev.currentImportantDateAt || ""), + }; + }); + }) + .catch(() => null); if (showLoading && typeof setStatus === "function") setStatus("requestModal", "", ""); } catch (error) { setRequestModal((prev) => ({ @@ -331,7 +399,7 @@ export function useRequestWorkspace(options) { if (typeof setStatus === "function") setStatus("requestModal", "Ошибка: " + error.message, "error"); } }, - [api, resolveAdminObjectSrc, setStatus, token, users] + [api, hydrateRequestMessageBodies, resolveAdminObjectSrc, setStatus, token, users] ); const refreshRequestModal = useCallback(async () => { @@ -509,25 +577,33 @@ export function useRequestWorkspace(options) { const loadOlderRequestMessages = useCallback(async () => { const requestId = String(requestModal.requestId || "").trim(); - const loadedCount = Number(requestModal.messagesLoadedCount || 0); + const cursor = getOldestMessageCursor(requestModal.messages); if (!api || !requestId || requestModal.messagesLoadingMore || !requestModal.messagesHasMore) return null; setRequestModal((prev) => ({ ...prev, messagesLoadingMore: true })); try { - const payload = await api( - "/api/admin/chat/requests/" + - requestId + - "/messages-window?before_count=" + - encodeURIComponent(String(loadedCount)) - ); + const query = new URLSearchParams({ include_body: "false" }); + if (cursor?.beforeId && cursor?.beforeCreatedAt) { + query.set("before_id", cursor.beforeId); + query.set("before_created_at", cursor.beforeCreatedAt); + } else { + query.set("before_count", String(Number(requestModal.messagesLoadedCount || 0))); + } + const payload = await api("/api/admin/chat/requests/" + requestId + "/messages-window?" + query.toString()); const nextMessages = normalizeMessageAuthors(payload?.rows || [], users); setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false, - messages: mergeRowsById(nextMessages, prev.messages), - messagesHasMore: Boolean(payload?.has_more), - messagesLoadedCount: Number(payload?.loaded_count || prev.messagesLoadedCount || 0), - messagesTotal: Number(payload?.total || prev.messagesTotal || 0), + ...(function () { + const merged = mergeRowsById(nextMessages, prev.messages); + return { + messages: merged, + messagesHasMore: Boolean(payload?.has_more), + messagesLoadedCount: merged.length, + messagesTotal: Number(prev.messagesTotal || merged.length || 0), + }; + })(), })); + void hydrateRequestMessageBodies(requestId, nextMessages); return payload || null; } catch (error) { setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false })); @@ -541,6 +617,7 @@ export function useRequestWorkspace(options) { requestModal.messagesLoadingMore, requestModal.requestId, setStatus, + hydrateRequestMessageBodies, users, ]); diff --git a/app/web/client.js b/app/web/client.js index 2bdd4a5..740f319 100644 --- a/app/web/client.js +++ b/app/web/client.js @@ -199,6 +199,8 @@ currentImportantDateAt, pendingStatusChangePreset, messages, + messagesHasMore, + messagesLoadingMore, attachments, messageDraft, selectedFiles, @@ -206,6 +208,7 @@ status, onMessageChange, onSendMessage, + onLoadOlderMessages, onFilesSelect, onRemoveSelectedFile, onClearSelectedFiles, @@ -1571,7 +1574,16 @@ disabled: loading || fileUploading, style: { position: "absolute", width: "1px", height: "1px", opacity: 0, pointerEvents: "none" } } - ), chatTab === "chat" ? /* @__PURE__ */ React.createElement(React.Fragment, null, /* @__PURE__ */ React.createElement("ul", { className: "simple-list request-modal-list request-chat-list", id: idMap.messagesList, ref: chatListRef }, chatTimelineItems.length ? chatTimelineItems.map( + ), chatTab === "chat" ? /* @__PURE__ */ React.createElement(React.Fragment, null, messagesHasMore ? /* @__PURE__ */ React.createElement("div", { className: "request-chat-history-actions" }, /* @__PURE__ */ React.createElement( + "button", + { + type: "button", + className: "btn secondary", + onClick: onLoadOlderMessages, + disabled: loading || fileUploading || messagesLoadingMore + }, + messagesLoadingMore ? "\u0417\u0430\u0433\u0440\u0443\u0437\u043A\u0430 \u0438\u0441\u0442\u043E\u0440\u0438\u0438..." : "\u041F\u043E\u043A\u0430\u0437\u0430\u0442\u044C \u043F\u0440\u0435\u0434\u044B\u0434\u0443\u0449\u0438\u0435 \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u044F" + )) : null, /* @__PURE__ */ React.createElement("ul", { className: "simple-list request-modal-list request-chat-list", id: idMap.messagesList, ref: chatListRef }, chatTimelineItems.length ? chatTimelineItems.map( (entry) => entry.type === "date" ? /* @__PURE__ */ React.createElement("li", { key: entry.key, className: "chat-date-divider" }, /* @__PURE__ */ React.createElement("span", null, entry.label)) : entry.type === "file" ? /* @__PURE__ */ React.createElement( "li", { @@ -1612,7 +1624,7 @@ } } : void 0 }, - String(entry.payload?.message_kind || "") === "REQUEST_DATA" ? /* @__PURE__ */ React.createElement(React.Fragment, null, /* @__PURE__ */ React.createElement("div", { className: "chat-request-data-head" }, "\u0417\u0430\u043F\u0440\u043E\u0441"), renderRequestDataMessageItems(entry.payload)) : /* @__PURE__ */ React.createElement(React.Fragment, null, serviceMessageContent?.title ? /* @__PURE__ */ React.createElement("div", { className: "chat-service-head" }, serviceMessageContent.title) : null, serviceMessageContent ? serviceMessageContent.text ? /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, serviceMessageContent.text) : null : /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, String(entry.payload?.body || ""))), + String(entry.payload?.message_kind || "") === "REQUEST_DATA" ? /* @__PURE__ */ React.createElement(React.Fragment, null, /* @__PURE__ */ React.createElement("div", { className: "chat-request-data-head" }, "\u0417\u0430\u043F\u0440\u043E\u0441"), renderRequestDataMessageItems(entry.payload)) : /* @__PURE__ */ React.createElement(React.Fragment, null, serviceMessageContent?.title ? /* @__PURE__ */ React.createElement("div", { className: "chat-service-head" }, serviceMessageContent.title) : null, serviceMessageContent ? serviceMessageContent.text ? /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, serviceMessageContent.text) : null : /* @__PURE__ */ React.createElement("p", { className: "chat-message-text" }, entry.payload?.body_loaded === false ? "\u0417\u0430\u0433\u0440\u0443\u0437\u043A\u0430 \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u044F..." : String(entry.payload?.body || ""))), (() => { if (String(entry.payload?.message_kind || "") === "REQUEST_DATA") return null; const messageId = String(entry.payload?.id || "").trim(); @@ -2237,6 +2249,10 @@ currentImportantDateAt: "", pendingStatusChangePreset: null, messages: [], + messagesHasMore: false, + messagesLoadingMore: false, + messagesLoadedCount: 0, + messagesTotal: 0, attachments: [], messageDraft: "", selectedFiles: [], @@ -2267,6 +2283,18 @@ }); return sortRowsByCreatedAt(Array.from(merged.values())); } + function getOldestMessageCursor(rows) { + const sorted = sortRowsByCreatedAt(Array.isArray(rows) ? rows : []); + const first = sorted[0]; + if (!first) return null; + const beforeId = String(first.id || "").trim(); + const beforeCreatedAt = String(first.created_at || first.updated_at || "").trim(); + if (!beforeId || !beforeCreatedAt) return null; + return { beforeId, beforeCreatedAt }; + } + function collectDeferredMessageIds(rows) { + return (Array.isArray(rows) ? rows : []).filter((row) => row && typeof row === "object" && row.body_loaded === false && String(row.id || "").trim()).map((row) => String(row.id).trim()); + } function StatusLine({ status }) { return /* @__PURE__ */ React.createElement("p", { className: "status" + (status?.kind ? " " + status.kind : "") }, status?.message || ""); } @@ -2701,6 +2729,37 @@ }); return completeData; }, [apiJson, buildStorageUploadError, requestModal.requestId, runUploadStepWithRetry]); + const hydratePublicMessageBodies = useCallback( + async (trackNumber, rows) => { + const track = String(trackNumber || "").trim().toUpperCase(); + const ids = collectDeferredMessageIds(rows); + if (!track || !ids.length) return null; + try { + const payload = await apiJson( + "/api/public/chat/requests/" + encodeURIComponent(track) + "/message-bodies", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ids }) + }, + "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0442\u0435\u043A\u0441\u0442\u044B \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u0439" + ); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; + if (!nextRows.length) return payload || null; + setRequestModal((prev) => { + if (String(prev.trackNumber || "").trim().toUpperCase() !== track) return prev; + return { + ...prev, + messages: mergeRowsById(prev.messages, nextRows) + }; + }); + return payload || null; + } catch (_) { + return null; + } + }, + [apiJson] + ); const loadRequestWorkspace = useCallback( async (trackNumber, showLoading) => { const track = String(trackNumber || "").trim().toUpperCase(); @@ -2710,7 +2769,11 @@ } const [requestData, messagesData, attachmentsData, invoicesData, statusRouteData, serviceRequestsData] = await Promise.all([ apiJson("/api/public/requests/" + encodeURIComponent(track), null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u043E\u0442\u043A\u0440\u044B\u0442\u044C \u0437\u0430\u044F\u0432\u043A\u0443"), - apiJson("/api/public/chat/requests/" + encodeURIComponent(track) + "/messages", null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u044F"), + apiJson( + "/api/public/chat/requests/" + encodeURIComponent(track) + "/messages-window?include_body=false", + null, + "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u044F" + ), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/attachments", null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0444\u0430\u0439\u043B\u044B"), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/invoices", null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0441\u0447\u0435\u0442\u0430"), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/status-route", null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u043C\u0430\u0440\u0448\u0440\u0443\u0442 \u0441\u0442\u0430\u0442\u0443\u0441\u043E\u0432"), @@ -2749,13 +2812,67 @@ availableStatuses: [], currentImportantDateAt: String(statusRouteData?.current_important_date_at || requestData?.important_date_at || ""), invoices, - messages: Array.isArray(messagesData) ? messagesData : [], + messages: Array.isArray(messagesData?.rows) ? messagesData.rows : [], + messagesHasMore: Boolean(messagesData?.has_more), + messagesLoadingMore: false, + messagesLoadedCount: Array.isArray(messagesData?.rows) ? messagesData.rows.length : 0, + messagesTotal: Number(messagesData?.total || (Array.isArray(messagesData?.rows) ? messagesData.rows.length : 0)), attachments: Array.isArray(attachmentsData) ? attachmentsData : [], fileUploading: false })); + void hydratePublicMessageBodies(track, Array.isArray(messagesData?.rows) ? messagesData.rows : []); }, - [apiJson] + [apiJson, hydratePublicMessageBodies] ); + const loadOlderPublicMessages = useCallback(async () => { + const track = String(activeTrack || requestModal.trackNumber || "").trim().toUpperCase(); + const cursor = getOldestMessageCursor(requestModal.messages); + if (!track || requestModal.messagesLoadingMore || !requestModal.messagesHasMore) return null; + setRequestModal((prev) => ({ ...prev, messagesLoadingMore: true })); + try { + const query = new URLSearchParams({ include_body: "false" }); + if (cursor?.beforeId && cursor?.beforeCreatedAt) { + query.set("before_id", cursor.beforeId); + query.set("before_created_at", cursor.beforeCreatedAt); + } else { + query.set("before_count", String(Number(requestModal.messagesLoadedCount || 0))); + } + const payload = await apiJson( + "/api/public/chat/requests/" + encodeURIComponent(track) + "/messages-window?" + query.toString(), + null, + "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0438\u0441\u0442\u043E\u0440\u0438\u044E \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u0439" + ); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; + setRequestModal((prev) => ({ + ...prev, + messagesLoadingMore: false, + ...(function() { + const merged = mergeRowsById(nextRows, prev.messages); + return { + messages: merged, + messagesHasMore: Boolean(payload?.has_more), + messagesLoadedCount: merged.length, + messagesTotal: Number(prev.messagesTotal || merged.length || 0) + }; + })() + })); + void hydratePublicMessageBodies(track, nextRows); + return payload || null; + } catch (error) { + setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false })); + setPageStatus(error?.message || "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0438\u0441\u0442\u043E\u0440\u0438\u044E \u0441\u043E\u043E\u0431\u0449\u0435\u043D\u0438\u0439", "error"); + return null; + } + }, [ + activeTrack, + apiJson, + requestModal.messagesHasMore, + requestModal.messagesLoadedCount, + requestModal.messagesLoadingMore, + requestModal.trackNumber, + setPageStatus, + hydratePublicMessageBodies + ]); const refreshRequestsList = useCallback(async () => { const data = await apiJson("/api/public/requests/my", null, "\u041D\u0435 \u0443\u0434\u0430\u043B\u043E\u0441\u044C \u0437\u0430\u0433\u0440\u0443\u0437\u0438\u0442\u044C \u0441\u043F\u0438\u0441\u043E\u043A \u0437\u0430\u044F\u0432\u043E\u043A"); const rows = Array.isArray(data?.rows) ? data.rows : []; @@ -2777,6 +2894,10 @@ statusRouteNodes: [], statusHistory: [], messages: [], + messagesHasMore: false, + messagesLoadingMore: false, + messagesLoadedCount: 0, + messagesTotal: 0, attachments: [], fileUploading: false, selectedFiles: [], @@ -2977,11 +3098,18 @@ const nextMessages = Array.isArray(payload?.messages) ? payload.messages : []; const nextAttachments = Array.isArray(payload?.attachments) ? payload.attachments : []; if (nextMessages.length || nextAttachments.length) { - setRequestModal((prev) => ({ - ...prev, - messages: mergeRowsById(prev.messages, nextMessages), - attachments: mergeRowsById(prev.attachments, nextAttachments) - })); + setRequestModal((prev) => { + const mergedMessages = mergeRowsById(prev.messages, nextMessages); + const previousCount = Array.isArray(prev.messages) ? prev.messages.length : 0; + const addedCount = Math.max(0, mergedMessages.length - previousCount); + return { + ...prev, + messages: mergedMessages, + messagesLoadedCount: Number(prev.messagesLoadedCount || previousCount) + addedCount, + messagesTotal: Number(prev.messagesTotal || previousCount) + addedCount, + attachments: mergeRowsById(prev.attachments, nextAttachments) + }; + }); } } return payload || { has_updates: false, typing: [], cursor: null }; @@ -3148,6 +3276,8 @@ currentImportantDateAt: requestModal.currentImportantDateAt || "", pendingStatusChangePreset: null, messages: requestModal.messages || [], + messagesHasMore: Boolean(requestModal.messagesHasMore), + messagesLoadingMore: Boolean(requestModal.messagesLoadingMore), attachments: requestModal.attachments || [], messageDraft: requestModal.messageDraft || "", selectedFiles: requestModal.selectedFiles || [], @@ -3155,6 +3285,7 @@ status, onMessageChange: updateMessageDraft, onSendMessage: submitMessage, + onLoadOlderMessages: loadOlderPublicMessages, onFilesSelect: appendFiles, onRemoveSelectedFile: removeFile, onClearSelectedFiles: clearFiles, diff --git a/app/web/client.jsx b/app/web/client.jsx index 21eff73..733d396 100644 --- a/app/web/client.jsx +++ b/app/web/client.jsx @@ -27,6 +27,22 @@ import { detectAttachmentPreviewKind, fmtShortDateTime, statusLabel } from "./ad return sortRowsByCreatedAt(Array.from(merged.values())); } + function getOldestMessageCursor(rows) { + const sorted = sortRowsByCreatedAt(Array.isArray(rows) ? rows : []); + const first = sorted[0]; + if (!first) return null; + const beforeId = String(first.id || "").trim(); + const beforeCreatedAt = String(first.created_at || first.updated_at || "").trim(); + if (!beforeId || !beforeCreatedAt) return null; + return { beforeId, beforeCreatedAt }; + } + + function collectDeferredMessageIds(rows) { + return (Array.isArray(rows) ? rows : []) + .filter((row) => row && typeof row === "object" && row.body_loaded === false && String(row.id || "").trim()) + .map((row) => String(row.id).trim()); + } + function StatusLine({ status }) { return

{status?.message || ""}

; } @@ -599,6 +615,38 @@ import { detectAttachmentPreviewKind, fmtShortDateTime, statusLabel } from "./ad return completeData; }, [apiJson, buildStorageUploadError, requestModal.requestId, runUploadStepWithRetry]); + const hydratePublicMessageBodies = useCallback( + async (trackNumber, rows) => { + const track = String(trackNumber || "").trim().toUpperCase(); + const ids = collectDeferredMessageIds(rows); + if (!track || !ids.length) return null; + try { + const payload = await apiJson( + "/api/public/chat/requests/" + encodeURIComponent(track) + "/message-bodies", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ids }), + }, + "Не удалось загрузить тексты сообщений" + ); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; + if (!nextRows.length) return payload || null; + setRequestModal((prev) => { + if (String(prev.trackNumber || "").trim().toUpperCase() !== track) return prev; + return { + ...prev, + messages: mergeRowsById(prev.messages, nextRows), + }; + }); + return payload || null; + } catch (_) { + return null; + } + }, + [apiJson] + ); + const loadRequestWorkspace = useCallback( async (trackNumber, showLoading) => { const track = String(trackNumber || "").trim().toUpperCase(); @@ -608,7 +656,11 @@ import { detectAttachmentPreviewKind, fmtShortDateTime, statusLabel } from "./ad } const [requestData, messagesData, attachmentsData, invoicesData, statusRouteData, serviceRequestsData] = await Promise.all([ apiJson("/api/public/requests/" + encodeURIComponent(track), null, "Не удалось открыть заявку"), - apiJson("/api/public/chat/requests/" + encodeURIComponent(track) + "/messages-window", null, "Не удалось загрузить сообщения"), + apiJson( + "/api/public/chat/requests/" + encodeURIComponent(track) + "/messages-window?include_body=false", + null, + "Не удалось загрузить сообщения" + ), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/attachments", null, "Не удалось загрузить файлы"), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/invoices", null, "Не удалось загрузить счета"), apiJson("/api/public/requests/" + encodeURIComponent(track) + "/status-route", null, "Не удалось загрузить маршрут статусов"), @@ -652,37 +704,49 @@ import { detectAttachmentPreviewKind, fmtShortDateTime, statusLabel } from "./ad messages: Array.isArray(messagesData?.rows) ? messagesData.rows : [], messagesHasMore: Boolean(messagesData?.has_more), messagesLoadingMore: false, - messagesLoadedCount: Number(messagesData?.loaded_count || 0), - messagesTotal: Number(messagesData?.total || 0), + messagesLoadedCount: Array.isArray(messagesData?.rows) ? messagesData.rows.length : 0, + messagesTotal: Number(messagesData?.total || (Array.isArray(messagesData?.rows) ? messagesData.rows.length : 0)), attachments: Array.isArray(attachmentsData) ? attachmentsData : [], fileUploading: false, })); + void hydratePublicMessageBodies(track, Array.isArray(messagesData?.rows) ? messagesData.rows : []); }, - [apiJson] + [apiJson, hydratePublicMessageBodies] ); const loadOlderPublicMessages = useCallback(async () => { const track = String(activeTrack || requestModal.trackNumber || "").trim().toUpperCase(); - const loadedCount = Number(requestModal.messagesLoadedCount || 0); + const cursor = getOldestMessageCursor(requestModal.messages); if (!track || requestModal.messagesLoadingMore || !requestModal.messagesHasMore) return null; setRequestModal((prev) => ({ ...prev, messagesLoadingMore: true })); try { + const query = new URLSearchParams({ include_body: "false" }); + if (cursor?.beforeId && cursor?.beforeCreatedAt) { + query.set("before_id", cursor.beforeId); + query.set("before_created_at", cursor.beforeCreatedAt); + } else { + query.set("before_count", String(Number(requestModal.messagesLoadedCount || 0))); + } const payload = await apiJson( - "/api/public/chat/requests/" + - encodeURIComponent(track) + - "/messages-window?before_count=" + - encodeURIComponent(String(loadedCount)), + "/api/public/chat/requests/" + encodeURIComponent(track) + "/messages-window?" + query.toString(), null, "Не удалось загрузить историю сообщений" ); + const nextRows = Array.isArray(payload?.rows) ? payload.rows : []; setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false, - messages: mergeRowsById(payload?.rows || [], prev.messages), - messagesHasMore: Boolean(payload?.has_more), - messagesLoadedCount: Number(payload?.loaded_count || prev.messagesLoadedCount || 0), - messagesTotal: Number(payload?.total || prev.messagesTotal || 0), + ...(function () { + const merged = mergeRowsById(nextRows, prev.messages); + return { + messages: merged, + messagesHasMore: Boolean(payload?.has_more), + messagesLoadedCount: merged.length, + messagesTotal: Number(prev.messagesTotal || merged.length || 0), + }; + })(), })); + void hydratePublicMessageBodies(track, nextRows); return payload || null; } catch (error) { setRequestModal((prev) => ({ ...prev, messagesLoadingMore: false })); @@ -697,6 +761,7 @@ import { detectAttachmentPreviewKind, fmtShortDateTime, statusLabel } from "./ad requestModal.messagesLoadingMore, requestModal.trackNumber, setPageStatus, + hydratePublicMessageBodies, ]); const refreshRequestsList = useCallback(async () => { diff --git a/context/19_performance_tracking_2026-03-16.md b/context/19_performance_tracking_2026-03-16.md index cac6aae..2575833 100644 --- a/context/19_performance_tracking_2026-03-16.md +++ b/context/19_performance_tracking_2026-03-16.md @@ -22,7 +22,7 @@ | PERF-04 | Собрать единый endpoint карточки заявки | in_progress | P0 | PERF-01 | | PERF-05 | Выделить узкие request-scoped endpoints для вложений и счетов | completed | P0 | PERF-04 | | PERF-06 | Переписать kanban на SQL-first фильтрацию/limit | in_progress | P0 | PERF-01, PERF-02 | -| PERF-07 | Ограничить initial chat payload и добавить догрузку истории | in_progress | P1 | PERF-03, PERF-04 | +| PERF-07 | Ограничить initial chat payload и добавить догрузку истории | completed | P1 | PERF-03, PERF-04 | | PERF-08 | Добавить нужные вспомогательные индексы и повторный profiling | planned | P1 | PERF-01 | ## PERF-01 @@ -80,6 +80,17 @@ - 2026-03-17: admin avatar proxy умеет по `variant=thumb` отдавать сжатый вариант и, если его еще нет, достраивать его на лету из оригинала; public featured staff URLs тоже переключены на `?variant=thumb` и умеют так же достраивать thumb на лету. - 2026-03-17: `workspace` упрощен server-side: убрано дублирующее `get_request_service() + db.get(Request)` внутри одного запроса, read-mark side effects сведены в один проход, `mark_admin_notifications_read` переведен на bulk update, `status_route` повторно использует уже загруженный `Request`. - 2026-03-17: контейнерные регрессы после avatar/workspace правок пройдены: `tests.test_uploads_s3`, `tests.test_featured_staff_public`, `tests.admin.test_lawyer_chat`. +- 2026-03-17: для admin UI первый `workspace` переведен в lean-режим: `/api/admin/requests/{id}/workspace?include_related=false` теперь отдает только заявку, чат-окно и базовый finance summary, а `attachments / invoices / status-route` догружаются фоном отдельными endpoint. Это режет количество SQL round-trip в критическом пути на проде с медленной БД. +- 2026-03-17: добавлена внутренняя инструментализация `workspace/status-route/serialize_messages` через `uvicorn.error`, чтобы видеть step-by-step ms в контейнерных логах без отдельного profiler. +- 2026-03-17: живой локальный профиль подтвердил bottleneck: почти весь `workspace` уходит в `messages_query_ms`, а не в `status-route` или дополнительных запросах. На чате в `2000` сообщений: при initial window `50` `messages_query_ms ~569 ms`, после уменьшения initial window до `20` `messages_query_ms ~239 ms`. +- 2026-03-17: корневая причина находится в загрузке `Message` rows с `EncryptedChatText`: дешифровка `Message.body` выполняется на materialize каждого ORM row и дает почти линейную стоимость по числу сообщений в initial window. +- 2026-03-17: chat crypto переработан без ослабления защиты: новые сообщения пишутся в `chatenc:v3` с `per-chat` data key, завернутым master chat key в `Request.extra_fields.chat_crypto`; чтение `v1/v2` сохранено для обратной совместимости. +- 2026-03-17: `Message.body` больше не auto-decrypt в ORM. Шифрование тела выполняется на `before_flush`, а дешифровка вынесена в `chat_secure_service` и вызывается только там, где действительно нужен текст сообщения. +- 2026-03-17: admin/public `messages-window` переведены на cursor-параметры `before_id + before_created_at` с сохранением fallback `before_count` для совместимости; UI admin/client переключен на cursor path. +- 2026-03-17: initial chat payload для admin workspace и public cabinet стал metadata-first: `workspace/messages-window` могут отдавать `body_loaded=false`, а тексты догружаются отдельным `message-bodies` batch endpoint только для реально показанных сообщений. +- 2026-03-17: добавлены новые endpoint `POST /api/admin/chat/requests/{id}/message-bodies` и `POST /api/public/chat/requests/{track}/message-bodies` с тем же RBAC/session-scope, что и чтение чата. +- 2026-03-17: reencrypt path обновлен под новый `v3` формат - `app/scripts/reencrypt_with_active_kid.py` теперь мигрирует legacy chat rows в request-scoped AEAD и одновременно заполняет `Request.extra_fields.chat_crypto`. +- 2026-03-17: контейнерный регресс нового chat stack пройден: `tests.test_reencrypt_with_active_kid`, `tests.test_public_cabinet`, `tests.admin.test_lawyer_chat`, `tests.test_invoices`, `tests.test_crypto_kid_rotation`, `tests.test_http_hardening` (`45 tests OK`). ## Дальше diff --git a/requirements.txt b/requirements.txt index b06008b..5e52e18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ python-multipart==0.0.22 smsaero-api-async Pillow==11.2.1 reportlab==4.2.2 +cryptography==45.0.7 diff --git a/tests/admin/test_lawyer_chat.py b/tests/admin/test_lawyer_chat.py index 1dc07f4..a83a7e9 100644 --- a/tests/admin/test_lawyer_chat.py +++ b/tests/admin/test_lawyer_chat.py @@ -384,24 +384,50 @@ class AdminLawyerChatTests(AdminUniversalCrudBase): self.assertEqual(own_workspace.status_code, 200) payload = own_workspace.json() self.assertEqual(str((payload.get("request") or {}).get("id")), own_id) - self.assertEqual(len(payload.get("messages") or []), 50) + self.assertEqual(len(payload.get("messages") or []), 20) self.assertTrue(bool(payload.get("messages_has_more"))) self.assertEqual(int(payload.get("messages_total") or 0), 55) - self.assertEqual(int(payload.get("messages_loaded_count") or 0), 50) + self.assertEqual(int(payload.get("messages_loaded_count") or 0), 20) + self.assertTrue(all(item.get("body_loaded") is False for item in (payload.get("messages") or []))) self.assertEqual(len(payload.get("attachments") or []), 1) self.assertIn("status_route", payload) self.assertIn("finance_summary", payload) + lean_workspace = self.client.get(f"/api/admin/requests/{own_id}/workspace?include_related=false", headers=headers) + self.assertEqual(lean_workspace.status_code, 200) + lean_payload = lean_workspace.json() + self.assertEqual(str((lean_payload.get("request") or {}).get("id")), own_id) + self.assertEqual(len(lean_payload.get("messages") or []), 20) + self.assertEqual(len(lean_payload.get("attachments") or []), 0) + self.assertEqual(len(lean_payload.get("invoices") or []), 0) + self.assertEqual((lean_payload.get("status_route") or {}).get("nodes") or [], []) + + body_batch = self.chat_client.post( + f"/api/admin/chat/requests/{own_id}/message-bodies", + headers=headers, + json={"ids": [item["id"] for item in (payload.get("messages") or [])[:3]]}, + ) + self.assertEqual(body_batch.status_code, 200) + self.assertEqual(len(body_batch.json().get("rows") or []), 3) + self.assertTrue(all(item.get("body_loaded") for item in (body_batch.json().get("rows") or []))) + + oldest_loaded = (payload.get("messages") or [])[0] + older_messages = self.chat_client.get( f"/api/admin/chat/requests/{own_id}/messages-window", headers=headers, - params={"before_count": 50, "limit": 10}, + params={ + "before_id": str(oldest_loaded.get("id") or ""), + "before_created_at": str(oldest_loaded.get("created_at") or ""), + "limit": 10, + "include_body": "false", + }, ) self.assertEqual(older_messages.status_code, 200) older_payload = older_messages.json() - self.assertEqual(len(older_payload.get("rows") or []), 5) - self.assertFalse(bool(older_payload.get("has_more"))) - self.assertEqual(int(older_payload.get("loaded_count") or 0), 55) + self.assertEqual(len(older_payload.get("rows") or []), 10) + self.assertTrue(bool(older_payload.get("has_more"))) + self.assertTrue(all(item.get("body_loaded") is False for item in (older_payload.get("rows") or []))) foreign_workspace = self.client.get(f"/api/admin/requests/{foreign_id}/workspace", headers=headers) self.assertEqual(foreign_workspace.status_code, 403) diff --git a/tests/test_crypto_kid_rotation.py b/tests/test_crypto_kid_rotation.py index e85f872..1f31455 100644 --- a/tests/test_crypto_kid_rotation.py +++ b/tests/test_crypto_kid_rotation.py @@ -12,7 +12,13 @@ os.environ.setdefault("S3_SECRET_KEY", "test") os.environ.setdefault("S3_BUCKET", "test") from app.core.config import settings -from app.services.chat_crypto import decrypt_message_body, encrypt_message_body, extract_message_kid +from app.services.chat_crypto import ( + decrypt_message_body, + decrypt_message_body_for_request, + encrypt_message_body, + encrypt_message_body_for_request, + extract_message_kid, +) from app.services.invoice_crypto import ( active_requisites_kid, decrypt_requisites, @@ -110,6 +116,24 @@ class CryptoKidRotationTests(unittest.TestCase): self.assertEqual(extract_message_kid(token), "k2") self.assertEqual(decrypt_message_body(token), "legacy message") + def test_chat_request_crypto_uses_per_chat_v3_format(self): + settings.DATA_ENCRYPTION_SECRET = "" + settings.DATA_ENCRYPTION_ACTIVE_KID = "k2" + settings.DATA_ENCRYPTION_KEYS = "k2=new-data-secret-bbbbbbbbbbbbbbbb" + settings.CHAT_ENCRYPTION_SECRET = "" + settings.CHAT_ENCRYPTION_ACTIVE_KID = "k2" + settings.CHAT_ENCRYPTION_KEYS = "k2=new-chat-secret-cccccccccccccccc" + + token, extra_fields, changed = encrypt_message_body_for_request("request scoped", request_extra_fields={}) + self.assertTrue(changed) + self.assertTrue(str(token).startswith("chatenc:v3:")) + self.assertEqual(extract_message_kid(token), "k2") + self.assertTrue(bool((extra_fields or {}).get("chat_crypto"))) + self.assertEqual( + decrypt_message_body_for_request(token, request_extra_fields=extra_fields), + "request scoped", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_invoices.py b/tests/test_invoices.py index 1673f30..0aede3c 100644 --- a/tests/test_invoices.py +++ b/tests/test_invoices.py @@ -27,6 +27,7 @@ from app.models.invoice import Invoice from app.models.message import Message from app.models.notification import Notification from app.models.request import Request +from app.services.chat_crypto import decrypt_message_body_for_request from app.services.invoice_crypto import decrypt_requisites @@ -190,7 +191,10 @@ class InvoiceApiTests(unittest.TestCase): self.assertEqual(decrypted["kpp"], "770001001") message = db.query(Message).filter(Message.request_id == UUID(self.request_a_id)).order_by(Message.created_at.desc()).first() self.assertIsNotNone(message) - self.assertEqual(message.body, "Счет на оплату") + self.assertEqual( + decrypt_message_body_for_request(message.body, request_extra_fields=row_request.extra_fields if (row_request := db.get(Request, UUID(self.request_a_id))) else {}), + "Счет на оплату", + ) attachment = ( db.query(Attachment) .filter(Attachment.request_id == UUID(self.request_a_id), Attachment.message_id == message.id) diff --git a/tests/test_public_cabinet.py b/tests/test_public_cabinet.py index 4a8f6b4..e9a835a 100644 --- a/tests/test_public_cabinet.py +++ b/tests/test_public_cabinet.py @@ -26,6 +26,7 @@ from app.models.attachment import Attachment from app.models.message import Message from app.models.notification import Notification from app.models.request import Request +from app.services.chat_crypto import decrypt_message_body_for_request from app.models.request_data_requirement import RequestDataRequirement from app.models.status_history import StatusHistory from app.services.chat_presence import clear_presence_for_tests, set_typing_presence @@ -217,9 +218,12 @@ class PublicCabinetTests(unittest.TestCase): self.assertIsNotNone(row) self.assertEqual(row.request_id, request_id) self.assertEqual(row.author_type, "CLIENT") - self.assertEqual(row.body, "Добрый день, есть вопрос по документам.") req = db.get(Request, request_id) self.assertIsNotNone(req) + self.assertEqual( + decrypt_message_body_for_request(row.body, request_extra_fields=req.extra_fields), + "Добрый день, есть вопрос по документам.", + ) self.assertEqual(req.responsible, "Клиент") self.assertTrue(req.lawyer_has_unread_updates) self.assertEqual(req.lawyer_unread_event_type, "MESSAGE") @@ -288,18 +292,33 @@ class PublicCabinetTests(unittest.TestCase): listed_window = self.chat_client.get( "/api/public/chat/requests/TRK-CHAT-001/messages-window", cookies=cookies, - params={"limit": 2}, + params={"limit": 2, "include_body": "false"}, ) self.assertEqual(listed_window.status_code, 200) window_payload = listed_window.json() self.assertEqual(len(window_payload.get("rows") or []), 2) self.assertTrue(bool(window_payload.get("has_more"))) - self.assertEqual(int(window_payload.get("loaded_count") or 0), 2) - self.assertEqual(int(window_payload.get("total") or 0), 5) + self.assertTrue(all(item.get("body_loaded") is False for item in (window_payload.get("rows") or []))) + + body_batch = self.chat_client.post( + "/api/public/chat/requests/TRK-CHAT-001/message-bodies", + cookies=cookies, + json={"ids": [item["id"] for item in (window_payload.get("rows") or [])]}, + ) + self.assertEqual(body_batch.status_code, 200) + self.assertEqual(len(body_batch.json().get("rows") or []), 2) + self.assertTrue(all(item.get("body_loaded") for item in (body_batch.json().get("rows") or []))) denied = self.chat_client.get("/api/public/chat/requests/TRK-CHAT-001/messages", cookies=self._public_cookies("TRK-OTHER")) self.assertEqual(denied.status_code, 404) + denied_batch = self.chat_client.post( + "/api/public/chat/requests/TRK-CHAT-001/message-bodies", + cookies=self._public_cookies("TRK-OTHER"), + json={"ids": [item["id"] for item in (window_payload.get("rows") or [])]}, + ) + self.assertEqual(denied_batch.status_code, 404) + def test_public_chat_marks_delivery_and_read_receipts_for_staff_messages(self): with self.SessionLocal() as db: req = Request( @@ -369,8 +388,11 @@ class PublicCabinetTests(unittest.TestCase): with self.SessionLocal() as db: raw_encrypted = db.execute(text("SELECT body FROM messages ORDER BY created_at DESC LIMIT 1")).scalar_one() - self.assertTrue(str(raw_encrypted).startswith("chatenc:")) + self.assertTrue(str(raw_encrypted).startswith("chatenc:v3:")) self.assertNotEqual(str(raw_encrypted), payload_body) + request_row = db.query(Request).filter(Request.track_number == "TRK-CHAT-ENC").first() + self.assertIsNotNone(request_row) + self.assertTrue(bool((request_row.extra_fields or {}).get("chat_crypto"))) listed = self.chat_client.get("/api/public/chat/requests/TRK-CHAT-ENC/messages", cookies=cookies) self.assertEqual(listed.status_code, 200) diff --git a/tests/test_reencrypt_with_active_kid.py b/tests/test_reencrypt_with_active_kid.py index 877886c..668f2e1 100644 --- a/tests/test_reencrypt_with_active_kid.py +++ b/tests/test_reencrypt_with_active_kid.py @@ -130,7 +130,7 @@ class ReencryptWithKidTests(unittest.TestCase): ) db.flush() db.execute( - text("UPDATE messages SET body = :body WHERE id = (SELECT id FROM messages ORDER BY created_at DESC LIMIT 1)"), + text("UPDATE messages SET body = :body WHERE rowid = (SELECT rowid FROM messages ORDER BY created_at DESC LIMIT 1)"), {"body": _legacy_chat_token("legacy body", old_secret)}, ) @@ -177,10 +177,13 @@ class ReencryptWithKidTests(unittest.TestCase): invoice_token = db.execute(text("SELECT payer_details_encrypted FROM invoices LIMIT 1")).scalar_one() admin_token = db.execute(text("SELECT totp_secret_encrypted FROM admin_users LIMIT 1")).scalar_one() message_token = db.execute(text("SELECT body FROM messages LIMIT 1")).scalar_one() + request_row = db.execute(text("SELECT extra_fields FROM requests LIMIT 1")).scalar_one() self.assertEqual(extract_requisites_kid(str(invoice_token)), "k2") self.assertEqual(extract_requisites_kid(str(admin_token)), "k2") + self.assertTrue(str(message_token).startswith("chatenc:v3:")) self.assertEqual(extract_message_kid(str(message_token)), "k2") + self.assertIn("chat_crypto", str(request_row)) if __name__ == "__main__":