#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$ROOT_DIR" DOMAIN="${DOMAIN:-ruakb.ru}" WWW_DOMAIN="${WWW_DOMAIN:-www.ruakb.ru}" SECOND_DOMAIN="${SECOND_DOMAIN:-ruakb.online}" SECOND_WWW_DOMAIN="${SECOND_WWW_DOMAIN:-www.ruakb.online}" LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-admin@ruakb.ru}" AUTO_CERT_INIT="${AUTO_CERT_INIT:-0}" SKIP_LOCAL_SMOKE="${SKIP_LOCAL_SMOKE:-0}" LOCAL_SMOKE_BASE_URL="${LOCAL_SMOKE_BASE_URL:-https://127.0.0.1}" LOCAL_SMOKE_CANDIDATES="${LOCAL_SMOKE_CANDIDATES:-${LOCAL_SMOKE_BASE_URL},https://localhost,http://127.0.0.1,http://localhost}" LOCAL_SMOKE_SKIP_DOCKER_CHECKS="${LOCAL_SMOKE_SKIP_DOCKER_CHECKS:-1}" LOCAL_SMOKE_DEBUG="${LOCAL_SMOKE_DEBUG:-0}" PROD_COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.prod.nginx.yml) CERT_COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.prod.nginx.yml -f docker-compose.prod.cert.yml) log() { echo "[SEC-AUDIT] $*" } warn() { echo "[SEC-AUDIT][WARN] $*" >&2 } fail() { echo "[SEC-AUDIT][ERROR] $*" >&2 exit 1 } file_missing() { [[ ! -f "$1" ]] } lower() { echo "${1:-}" | tr '[:upper:]' '[:lower:]' } read_env_value() { local file="$1" local key="$2" if [[ ! -f "$file" ]]; then echo "" return 0 fi grep -E "^${key}=" "$file" | tail -n1 | cut -d= -f2- || true } upsert_env_value() { local file="$1" local key="$2" local value="$3" local tmp tmp="$(mktemp)" awk -v k="$key" -v v="$value" ' BEGIN { done = 0 } $0 ~ ("^" k "=") { print k "=" v; done = 1; next } { print } END { if (!done) print k "=" v } ' "$file" > "$tmp" mv "$tmp" "$file" } is_prod_env() { local value value="$(lower "${1:-}")" [[ "$value" == "prod" || "$value" == "production" ]] } is_valid_samesite() { local value value="$(lower "${1:-}")" [[ "$value" == "lax" || "$value" == "strict" || "$value" == "none" ]] } is_truthy() { local value value="$(lower "${1:-}")" [[ "$value" == "true" || "$value" == "1" || "$value" == "yes" || "$value" == "on" ]] } ensure_env_file() { if [[ ! -f ".env" ]]; then if [[ -f ".env.prod" ]]; then cp .env.prod .env chmod 600 .env log ".env was missing -> restored from .env.prod" elif [[ -f ".env.production" ]]; then cp .env.production .env chmod 600 .env log ".env was missing -> restored from .env.production" else fail "Cannot build .env automatically: missing .env, .env.prod and .env.production" fi fi local app_env cookie_secure current_samesite strict_origin target_samesite local changed=0 app_env="$(read_env_value ".env" "APP_ENV")" cookie_secure="$(read_env_value ".env" "PUBLIC_COOKIE_SECURE")" current_samesite="$(read_env_value ".env" "PUBLIC_COOKIE_SAMESITE")" strict_origin="$(read_env_value ".env" "PUBLIC_STRICT_ORIGIN_CHECK")" target_samesite="$(lower "$current_samesite")" if ! is_valid_samesite "$target_samesite"; then local fallback_samesite fallback_samesite="$(lower "$(read_env_value ".env.prod" "PUBLIC_COOKIE_SAMESITE")")" if is_valid_samesite "$fallback_samesite"; then target_samesite="$fallback_samesite" else target_samesite="lax" fi fi if ! is_prod_env "$app_env"; then changed=1 fi if ! is_truthy "$cookie_secure"; then changed=1 fi if ! is_valid_samesite "$current_samesite"; then changed=1 fi if ! is_truthy "$strict_origin"; then changed=1 fi if [[ "$changed" == "1" ]]; then local backup_file backup_file=".env.backup.$(date +%Y%m%d-%H%M%S)" cp .env "$backup_file" chmod 600 "$backup_file" upsert_env_value ".env" "APP_ENV" "prod" upsert_env_value ".env" "PUBLIC_COOKIE_SECURE" "true" upsert_env_value ".env" "PUBLIC_COOKIE_SAMESITE" "$target_samesite" upsert_env_value ".env" "PUBLIC_STRICT_ORIGIN_CHECK" "true" chmod 600 .env warn ".env updated in-place for production smoke checks (backup: ${backup_file})" else log ".env found (APP_ENV=$(lower "$app_env"), PUBLIC_COOKIE_SAMESITE=$(lower "$current_samesite"))" fi } ensure_minio_tls_bundle() { if file_missing "deploy/tls/minio/public.crt" || file_missing "deploy/tls/minio/private.key" || file_missing "deploy/tls/minio/ca.crt"; then log "MinIO TLS bundle is missing -> generating" MINIO_TLS_OVERWRITE=true ./scripts/ops/minio_tls_bootstrap.sh return 0 fi if ! openssl x509 -in "deploy/tls/minio/ca.crt" -noout >/dev/null 2>&1; then log "MinIO CA certificate is invalid -> regenerating" MINIO_TLS_OVERWRITE=true ./scripts/ops/minio_tls_bootstrap.sh return 0 fi if ! openssl x509 -in "deploy/tls/minio/public.crt" -noout >/dev/null 2>&1; then log "MinIO public certificate is invalid -> regenerating" MINIO_TLS_OVERWRITE=true ./scripts/ops/minio_tls_bootstrap.sh else log "MinIO TLS bundle present" fi } ensure_compose_files() { file_missing "docker-compose.prod.nginx.yml" && fail "Missing docker-compose.prod.nginx.yml" file_missing "docker-compose.prod.cert.yml" && fail "Missing docker-compose.prod.cert.yml" file_missing "frontend/nginx.prod.conf" && fail "Missing frontend/nginx.prod.conf" file_missing "deploy/nginx/edge-http-only.conf" && fail "Missing deploy/nginx/edge-http-only.conf" file_missing "deploy/nginx/edge-https.conf" && fail "Missing deploy/nginx/edge-https.conf" log "Compose/nginx files present" } stack_up_and_migrate() { log "Starting core infra services" "${PROD_COMPOSE[@]}" up -d --build --remove-orphans --force-recreate db redis minio clamav log "Starting app services (backend/chat/email/worker/beat)" "${PROD_COMPOSE[@]}" up -d --build --remove-orphans --force-recreate backend chat-service email-service worker beat security-scheduler log "Waiting app services to become healthy" wait_service_healthy "backend" 60 wait_service_healthy "chat-service" 60 wait_service_healthy "email-service" 60 wait_service_healthy "security-scheduler" 90 # Force recreate frontend/edge after chat/backend to avoid stale DNS upstream cache in nginx. log "Starting/recreating frontend and edge" "${PROD_COMPOSE[@]}" up -d --build --remove-orphans --force-recreate frontend edge log "Applying migrations" "${PROD_COMPOSE[@]}" exec -T backend alembic upgrade head } run_security_preflight() { log "Running production security preflight (app-level config validation)" "${PROD_COMPOSE[@]}" run --rm --no-deps backend python - <<'PY' from app.core.config import validate_production_security_or_raise validate_production_security_or_raise("prod-security-audit") print("production security config validation: ok") PY } wait_service_healthy() { local service="$1" local max_attempts="${2:-60}" local attempt=1 local cid="" local status="" while (( attempt <= max_attempts )); do cid="$("${PROD_COMPOSE[@]}" ps -q "$service" 2>/dev/null || true)" if [[ -n "$cid" ]]; then status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid" 2>/dev/null || true)" case "$status" in healthy|running) return 0 ;; esac fi sleep 2 attempt=$((attempt + 1)) done fail "Service '${service}' did not become healthy in time (last_status=${status:-unknown})" } run_local_smoke() { if [[ "$SKIP_LOCAL_SMOKE" == "1" ]]; then log "Skipping local smoke checks (SKIP_LOCAL_SMOKE=1)" return 0 fi log "Running local smoke checks (candidates: ${LOCAL_SMOKE_CANDIDATES})" local max_attempts="${LOCAL_SMOKE_MAX_ATTEMPTS:-24}" local sleep_seconds="${LOCAL_SMOKE_SLEEP_SECONDS:-5}" local attempt=1 local candidate local ok=0 local debug_log debug_log="$(mktemp)" trap 'rm -f -- "${debug_log:-}"; trap - RETURN' RETURN while (( attempt <= max_attempts )); do ok=0 IFS=',' read -r -a _urls <<< "$LOCAL_SMOKE_CANDIDATES" for candidate in "${_urls[@]}"; do candidate="$(echo "$candidate" | xargs)" [[ -z "$candidate" ]] && continue : > "$debug_log" local health_rc smoke_rc if CHECK_CHAT_HEALTH_SKIP_DOCKER_CHECKS="$LOCAL_SMOKE_SKIP_DOCKER_CHECKS" ./scripts/ops/check_chat_health.sh "$candidate" >"$debug_log" 2>&1; then health_rc=0 else health_rc=$? fi if [[ $health_rc -ne 0 ]]; then if [[ "$LOCAL_SMOKE_DEBUG" == "1" || "$attempt" -eq "$max_attempts" ]]; then warn "local smoke health check failed for ${candidate} (rc=${health_rc})" sed -n '1,120p' "$debug_log" >&2 || true fi continue fi : > "$debug_log" if SECURITY_SMOKE_SKIP_DOCKER_CHECKS="$LOCAL_SMOKE_SKIP_DOCKER_CHECKS" ./scripts/ops/security_smoke.sh "$candidate" >"$debug_log" 2>&1; then smoke_rc=0 else smoke_rc=$? fi if [[ $smoke_rc -ne 0 ]]; then if [[ "$LOCAL_SMOKE_DEBUG" == "1" || "$attempt" -eq "$max_attempts" ]]; then warn "local smoke security checks failed for ${candidate} (rc=${smoke_rc})" sed -n '1,160p' "$debug_log" >&2 || true fi continue fi if [[ $health_rc -eq 0 && $smoke_rc -eq 0 ]]; then log "Local smoke checks passed via ${candidate} (attempt ${attempt}/${max_attempts})" ok=1 break fi done if [[ "$ok" == "1" ]]; then return 0 fi warn "Local smoke not ready yet (attempt ${attempt}/${max_attempts}), retrying in ${sleep_seconds}s" sleep "$sleep_seconds" attempt=$((attempt + 1)) done fail "Local smoke checks failed after ${max_attempts} attempts (candidates: ${LOCAL_SMOKE_CANDIDATES})" } run_domain_quick_health_wait() { local url="$1" local max_attempts="${DOMAIN_HEALTH_MAX_ATTEMPTS:-24}" local sleep_seconds="${DOMAIN_HEALTH_SLEEP_SECONDS:-5}" local attempt=1 while (( attempt <= max_attempts )); do if https_health_ok "$url"; then return 0 fi warn "HTTPS health not ready for ${url} (attempt ${attempt}/${max_attempts}), retrying in ${sleep_seconds}s" sleep "$sleep_seconds" attempt=$((attempt + 1)) done return 1 } run_domain_smoke() { local domain="$1" [[ -z "$domain" ]] && return 0 local url="https://${domain}" if ! run_domain_quick_health_wait "$url"; then if [[ "$AUTO_CERT_INIT" == "1" ]]; then cert_bootstrap run_domain_quick_health_wait "$url" || fail "HTTPS health still failing after cert bootstrap: ${url}/health" else fail "HTTPS health check failed: ${url}/health (set AUTO_CERT_INIT=1 to auto-bootstrap certs)" fi fi log "Running security smoke for $url" ./scripts/ops/security_smoke.sh "$url" >/dev/null log "Domain security smoke passed: $url" } https_health_ok() { local url="$1" local code code="$(curl -k -sS -o /dev/null -w "%{http_code}" "${url%/}/health" || true)" [[ "$code" == "200" ]] } cert_bootstrap() { log "AUTO_CERT_INIT=1 and https health failed -> running cert bootstrap" "${CERT_COMPOSE[@]}" up -d --build db redis minio backend chat-service email-service worker beat frontend edge "${CERT_COMPOSE[@]}" run --rm certbot certonly --webroot -w /var/www/certbot \ --email "$LETSENCRYPT_EMAIL" --agree-tos --no-eff-email --non-interactive --expand \ -d "$DOMAIN" -d "$WWW_DOMAIN" -d "$SECOND_DOMAIN" -d "$SECOND_WWW_DOMAIN" "${PROD_COMPOSE[@]}" up -d --build edge } run_incident_report() { log "Generating incident checklist snapshot" ./scripts/ops/incident_checklist.sh \ --severity LOW \ --category MONITORING_ALERT \ --summary "Scheduled production security audit completed" } print_summary() { log "Collecting final status" "${PROD_COMPOSE[@]}" ps local latest_security_report latest_security_report="$(ls -1t reports/security/security-smoke-*.md 2>/dev/null | head -n 1 || true)" local latest_incident_report latest_incident_report="$(ls -1t reports/incidents/incident-*.md 2>/dev/null | head -n 1 || true)" [[ -n "$latest_security_report" ]] && log "Latest security smoke report: $latest_security_report" [[ -n "$latest_incident_report" ]] && log "Latest incident checklist: $latest_incident_report" } main() { ensure_compose_files ensure_env_file ensure_minio_tls_bundle stack_up_and_migrate run_security_preflight run_local_smoke run_domain_smoke "$DOMAIN" run_domain_smoke "$SECOND_DOMAIN" run_incident_report print_summary log "Production security audit completed successfully" } main "$@"