commit 8b0c9783373fb7d02bb6c1f4d2c3e7d251c1e630 Author: TronoSfera <119615520+TronoSfera@users.noreply.github.com> Date: Sun May 17 20:22:35 2026 +0300 feat(initial): Backupy agent + backupy-decrypt CLI Source ports from the TronoSfera/backupy-cloud monorepo: - apps/agent/ — Go agent (WSS client, persistent queue, Docker discovery, 5 DB drivers: PG/MySQL/Mongo/Redis/SQLite, pre/post hooks, Prometheus metrics) - apps/backupy-decrypt/ — standalone CLI for client-side decryption - packages/proto/ — protobuf wire format (generated .pb.go committed so the repo builds without protoc) - docs/ — agent spec + wire-protocol contract Apache-2.0 license. Image published to ghcr.io/tronosfera/backupy-agent on every v* tag via .github/workflows/release.yml (multi-arch amd64+arm64). diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4919698 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,78 @@ +# Builds the agent as a multi-arch image and publishes to GHCR on every +# tag matching `v*`. The image gets three tags: +# ghcr.io/tronosfera/backupy-agent:vX.Y.Z (always) +# ghcr.io/tronosfera/backupy-agent:vX.Y (always) +# ghcr.io/tronosfera/backupy-agent:latest (only on non-prerelease tags) +# +# Uses the workflow's GITHUB_TOKEN — no extra secrets needed. + +name: release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + tag: + description: "Tag to (re-)build (e.g. v0.1.0)" + required: true + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Derive metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/backupy-agent + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }} + labels: | + org.opencontainers.image.title=backupy-agent + org.opencontainers.image.description=Open-source backup agent for backupy.tronosfera.ru + org.opencontainers.image.licenses=Apache-2.0 + org.opencontainers.image.source=https://github.com/${{ github.repository }} + + - name: Build & push + uses: docker/build-push-action@v6 + with: + context: . + file: apps/agent/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ github.ref_name }} + COMMIT=${{ github.sha }} + BUILD_DATE=${{ github.event.repository.updated_at }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..33dc24a --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Go build artifacts +*.test +*.out +/bin/ +/dist/ +/build/ + +# IDE +.idea/ +.vscode/ +.DS_Store +*.swp + +# Local secrets / runtime +.env +.env.local +.env.*.local +*.pem +*.key +!*.pub.key +secrets/ + +# Logs +*.log +logs/ + +# State (BoltDB queue / WAL) +*.db +*.bolt +state/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1699b26 --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + Copyright 2026 Backupy Agent contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7669ef6 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# Backupy Agent + +Open-source backup agent for the [Backupy](https://backupy.tronosfera.ru) backup-as-a-service platform. + +- Auto-discovers databases inside your Docker stack (PostgreSQL, MySQL, MongoDB, Redis, SQLite) +- Streams dumps to your cloud bucket, encrypted client-side with AES-256-GCM +- Keeps a persistent local queue so a brief network blip can't lose a run +- Talks to the cloud over WebSocket; no inbound ports on your host +- Apache-2.0 licensed; runs on the source code in this repo, end to end + +## Quick start + +1. Sign up at https://backupy.tronosfera.ru +2. Create an agent in **Dashboard → Agents → Add agent**. Copy the one-time key. +3. Add the snippet below to your `docker-compose.yml` (alongside the database you want to back up): + +```yaml +services: + backupy-agent: + image: ghcr.io/tronosfera/backupy-agent:v0.1.0 + restart: unless-stopped + environment: + BACKUPY_SERVER_URL: wss://backupy.tronosfera.ru/agents/connect + BACKUPY_AGENT_KEY: ${BACKUPY_AGENT_KEY} + volumes: + # Read-only socket for Docker discovery — required if you want + # auto-detection of running containers (recommended). + - /var/run/docker.sock:/var/run/docker.sock:ro + # Persistent state (BoltDB queue + last-seen offsets). + - backupy_agent:/var/lib/backupy + +volumes: + backupy_agent: +``` + +Put the key in your `.env`: + +``` +BACKUPY_AGENT_KEY=bk_agent_xxxxxxxxxxxxxxxxxxxxxxxx +``` + +``` +docker compose up -d backupy-agent +``` + +The agent connects, registers, and shows up in your dashboard. Configure the first backup job from there. + +## Build from source + +``` +make proto # regenerate Go bindings from packages/proto/ +make agent # builds the binary at apps/agent/bin/backupy-agent +make agent-image # builds the Docker image as backupy-agent:dev +``` + +## What's in this repo + +| Path | What | +|---|---| +| `apps/agent/` | The Go agent itself (cmd + internal). Multi-arch Docker image is published to `ghcr.io/tronosfera/backupy-agent`. | +| `apps/backupy-decrypt/` | Standalone CLI to decrypt a downloaded backup locally. You never need to upload the decryption key — it's handed to you in a one-time JWT signed by the server. | +| `packages/proto/` | Protobuf wire format between agent and server. The generated Go files (`.pb.go`) are committed so the repo builds clean without `protoc`. | +| `docs/` | Subset of the architectural docs that apply to the agent + the wire protocol. | + +## Releasing + +Push a tag matching `v*` to trigger the GHCR release workflow (`.github/workflows/release.yml`). It builds multi-arch (`linux/amd64` + `linux/arm64`) and publishes: + +- `ghcr.io/tronosfera/backupy-agent:vX.Y.Z` +- `ghcr.io/tronosfera/backupy-agent:vX.Y` +- `ghcr.io/tronosfera/backupy-agent:latest` (only for non-pre-release tags) + +## Security + +The agent has read-only access to the Docker socket (when mounted) and SHELL exec rights inside its own container for `mongodump`, `pg_dump`, etc. It never reaches outside your host except to: + +- `wss://backupy.tronosfera.ru/agents/connect` — control channel +- Presigned S3 PUT URLs returned by the server — to upload encrypted dump chunks + +If you set `BACKUPY_DISABLE_DISCOVERY=true`, the agent ignores the Docker socket and operates purely on explicit job configuration. + +## License + +Apache-2.0. See `LICENSE`. diff --git a/apps/agent/.gitignore b/apps/agent/.gitignore new file mode 100644 index 0000000..7f1f2f8 --- /dev/null +++ b/apps/agent/.gitignore @@ -0,0 +1,5 @@ +bin/ +coverage.out +*.test +.env +.env.local diff --git a/apps/agent/.gitkeep b/apps/agent/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/agent/Dockerfile b/apps/agent/Dockerfile new file mode 100644 index 0000000..ec1542c --- /dev/null +++ b/apps/agent/Dockerfile @@ -0,0 +1,104 @@ +# syntax=docker/dockerfile:1.7 +# +# Backupy agent image — multi-stage, multi-arch (linux/amd64, linux/arm64). +# Bundles client binaries the agent shells out to during dump: +# pg_dump, mysqldump, mongodump, redis-cli, sqlite3. +# +# Built from the REPO ROOT so the protobuf generated code in +# packages/proto/gen/go/backupv1 is in build context (referenced by the +# `replace` directive in apps/agent/go.mod). +# +# Example local build (multi-arch): +# docker buildx build \ +# --platform linux/amd64,linux/arm64 \ +# --build-arg VERSION=$(git describe --tags --always) \ +# --build-arg COMMIT=$(git rev-parse --short HEAD) \ +# --build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ +# -f apps/agent/Dockerfile \ +# -t ghcr.io/tronosfera/backupy-agent:dev . + +# ----------------------------------------------------------------------------- +# Stage 1 — Go build +# ----------------------------------------------------------------------------- +FROM --platform=$BUILDPLATFORM golang:1.23-alpine AS builder +RUN apk add --no-cache git + +WORKDIR /src + +# Module files first so dep download is cached independently of source. +COPY apps/agent/go.mod apps/agent/go.sum* ./apps/agent/ +COPY packages/proto/gen/go/backupv1 ./packages/proto/gen/go/backupv1 + +RUN --mount=type=cache,target=/go/pkg/mod \ + cd apps/agent && go mod download + +# Source last for better cache hit rate. +COPY apps/agent ./apps/agent + +ARG TARGETOS +ARG TARGETARCH +ARG VERSION=dev +ARG COMMIT=none +ARG BUILD_DATE=unknown + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + cd apps/agent && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build \ + -trimpath \ + -ldflags "-s -w \ + -X github.com/backupy/backupy/apps/agent/internal/version.Version=${VERSION} \ + -X github.com/backupy/backupy/apps/agent/internal/version.Commit=${COMMIT} \ + -X github.com/backupy/backupy/apps/agent/internal/version.BuildDate=${BUILD_DATE}" \ + -o /out/agent ./cmd/agent + +# ----------------------------------------------------------------------------- +# Stage 2 — Source the database client binaries from Alpine packages. +# Doing this in a separate stage keeps the apk cache out of the final image. +# ----------------------------------------------------------------------------- +FROM alpine:3.20 AS dbclients +RUN apk add --no-cache \ + postgresql16-client \ + mariadb-client \ + mongodb-tools \ + redis \ + sqlite + +# ----------------------------------------------------------------------------- +# Stage 3 — runtime (alpine, non-root uid 1000) +# ----------------------------------------------------------------------------- +FROM alpine:3.20 AS runtime + +RUN apk add --no-cache \ + ca-certificates \ + tzdata \ + libpq \ + mariadb-connector-c \ + openssl \ + sqlite-libs \ + && addgroup -S backupy -g 1000 \ + && adduser -S backupy -G backupy -u 1000 + +ARG VERSION=dev +ARG COMMIT=none +LABEL org.opencontainers.image.title="backupy-agent" \ + org.opencontainers.image.description="Open-source backup agent for backupy.tronosfera.ru" \ + org.opencontainers.image.source="https://github.com/TronoSfera/backupy-agent" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${COMMIT}" + +COPY --from=builder /out/agent /usr/local/bin/agent +COPY --from=dbclients /usr/bin/pg_dump /usr/bin/pg_dump +COPY --from=dbclients /usr/bin/mysqldump /usr/bin/mysqldump +COPY --from=dbclients /usr/bin/mongodump /usr/bin/mongodump +COPY --from=dbclients /usr/bin/redis-cli /usr/bin/redis-cli +COPY --from=dbclients /usr/bin/sqlite3 /usr/bin/sqlite3 + +USER backupy:backupy + +VOLUME ["/var/lib/backupy"] + +ENTRYPOINT ["/usr/local/bin/agent"] +CMD ["run"] diff --git a/apps/agent/Makefile b/apps/agent/Makefile new file mode 100644 index 0000000..3b03e8a --- /dev/null +++ b/apps/agent/Makefile @@ -0,0 +1,85 @@ +# Backupy agent — local developer targets. +# +# Quick start: +# make tidy # populate go.sum (requires `make proto` from repo root first) +# make test # unit tests with -race +# make build # local arch binary into bin/agent +# make image # docker build (local arch) +# +# All targets are phony — there are no on-disk artefacts to track besides +# bin/agent which is rebuilt on every `make build`. + +VERSION ?= dev +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo none) +BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +PKG_PREFIX := github.com/backupy/backupy/apps/agent/internal/version +LDFLAGS := -s -w \ + -X $(PKG_PREFIX).Version=$(VERSION) \ + -X $(PKG_PREFIX).Commit=$(COMMIT) \ + -X $(PKG_PREFIX).BuildDate=$(BUILD_DATE) + +GO ?= go +PKGS := ./... +BIN_DIR := bin +IMAGE ?= backupy/agent +IMG_TAG ?= dev + +# Repo root for docker buildx (build context includes packages/proto). +REPO_ROOT := $(shell git rev-parse --show-toplevel 2>/dev/null || echo ../..) + +.PHONY: tidy build build-linux-amd64 build-linux-arm64 test vet lint fmt image image-multiarch clean + +tidy: + $(GO) mod tidy + +build: + mkdir -p $(BIN_DIR) + CGO_ENABLED=0 $(GO) build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/agent ./cmd/agent + +build-linux-amd64: + mkdir -p $(BIN_DIR) + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + $(GO) build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/agent-linux-amd64 ./cmd/agent + +build-linux-arm64: + mkdir -p $(BIN_DIR) + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 \ + $(GO) build -trimpath -ldflags "$(LDFLAGS)" -o $(BIN_DIR)/agent-linux-arm64 ./cmd/agent + +test: + $(GO) test -race -cover $(PKGS) + +vet: + $(GO) vet $(PKGS) + +# Requires golangci-lint installed locally; CI installs it separately. +lint: + golangci-lint run $(PKGS) + +fmt: + $(GO) fmt $(PKGS) + +# Single-arch local image build. Run from anywhere; uses repo root context. +image: + docker build \ + --build-arg VERSION=$(VERSION) \ + --build-arg COMMIT=$(COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -f $(REPO_ROOT)/apps/agent/Dockerfile \ + -t $(IMAGE):$(IMG_TAG) \ + $(REPO_ROOT) + +# Multi-arch via buildx. Requires `docker buildx create --use` once. +image-multiarch: + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --build-arg VERSION=$(VERSION) \ + --build-arg COMMIT=$(COMMIT) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + -f $(REPO_ROOT)/apps/agent/Dockerfile \ + -t $(IMAGE):$(IMG_TAG) \ + $(REPO_ROOT) + +clean: + rm -rf $(BIN_DIR) diff --git a/apps/agent/README.md b/apps/agent/README.md new file mode 100644 index 0000000..1bffd6f --- /dev/null +++ b/apps/agent/README.md @@ -0,0 +1,139 @@ +# backupy-agent + +The Backupy agent — open-source (MIT) Docker service that runs alongside +your application and ships encrypted database backups to S3. + +> Full spec: [`docs/03-agent-spec.md`](../../docs/03-agent-spec.md). +> Wire protocol: [`docs/07-api-contract.md`](../../docs/07-api-contract.md). + +## Current status + +This directory is the **D-01 + D-03 (partial) skeleton**. What works today: + +- Compiles to a tiny static binary (`make build`). +- Loads & validates bootstrap config from env. +- Opens an encrypted-at-rest BoltDB state file (AES-256-GCM, HKDF from key). +- Cobra-based CLI: `run`, `version`, `health-check`, `dump-state`. +- Distroless multi-arch Dockerfile (< 50 MB image, non-root). + +What's stubbed: + +- WSS protocol exchange — skeleton only (full impl: **D-02**). +- Register / Heartbeat — interfaces in place (**D-04**). +- Docker socket discovery — interface only (**D-05**). +- Backup drivers (pg_dump, mysqldump, …) — interfaces only (**D-06+**). +- Auto-update — not started (**D-15**). + +## Quickstart (Docker) + +```yaml +services: + backup-agent: + image: backupy/agent:1 + environment: + BACKUP_SERVER_URL: https://api.backupy.ru + BACKUP_AGENT_KEY: ${BACKUP_KEY} + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - backup-agent-state:/var/lib/backup-agent + restart: unless-stopped + +volumes: + backup-agent-state: +``` + +## Env vars + +| Var | Required | Default | Notes | +|---|---|---|---| +| `BACKUP_SERVER_URL` | yes | `https://api.backupy.ru` | Must be `https://` (override with `BACKUP_DEV_ALLOW_INSECURE=true` for local dev). | +| `BACKUP_AGENT_KEY` | yes | — | Format `bkpy_(live\|test)_<32 alnum>`. Never logged. | +| `BACKUP_STATE_DIR` | no | `/var/lib/backup-agent` | Volume-mounted. Must be writable by uid 65532 (distroless `nonroot`). | +| `BACKUP_LOG_LEVEL` | no | `info` | `trace`/`debug`/`info`/`warn`/`error`. | +| `BACKUP_DOCKER_SOCKET` | no | `/var/run/docker.sock` | Mounted read-only. | +| `BACKUP_DEV_ALLOW_INSECURE` | no | `false` | Allows `http://` server URL — dev only. | + +Everything else (targets, schedules, retention, S3 creds, hooks) comes from +the server via `ConfigUpdate` after registration. + +## CLI + +```text +agent run # default; starts the service loop +agent version # print version / commit / build date (--json for machine output) +agent health-check # used by Docker HEALTHCHECK; exits 0 when healthy +agent dump-state # debug: pretty-print state.db as JSON + # add --allow-secrets to include decrypted payloads +``` + +## Filesystem layout + +``` +/var/lib/backup-agent/ +└── state.db # BoltDB, AES-256-GCM encrypted values (HKDF from agent key) +``` + +Buckets inside `state.db`: + +| Bucket | Contents | +|---|---| +| `config` | last applied `AgentConfig` + version | +| `queue` | pending `RunBackup` envelopes keyed by `run_id` | +| `registry` | `session_id`, last server time, last heartbeat | +| `logs_buffer` | rate-limited `LogEvent` buffer when server unreachable | + +## Build + +```sh +# Generate protobuf bindings first (from repo root). +make proto + +# Local binary +cd apps/agent +make tidy +make build # → bin/agent + +# Local docker image (uses repo root as build context) +make image + +# Multi-arch image +make image-multiarch +``` + +> **Important:** `make proto` from the repo root must run at least once +> before `go build` succeeds. The agent's `go.mod` uses a `replace` +> directive pointing at `packages/proto/gen/go/v1`, which is created by +> `buf generate` — see [`packages/proto/README.md`](../../packages/proto/README.md). + +## Tests + +```sh +make test # go test -race -cover ./... +``` + +Test packages that pass standalone (no proto codegen needed): + +- `internal/config` — env validation, agent-key regex, state-dir probe. +- `internal/state` — BoltDB roundtrip, AES-GCM encryption-at-rest check. +- `internal/logging` — level parsing, secret redaction. +- `internal/wss` — exponential backoff schedule + jitter band. + +Packages that require generated proto code to compile: + +- `internal/proto` +- `internal/wss/client.go` +- `cmd/agent/run.go` + +## Security notes + +- TLS 1.3 to all server endpoints (enforced by `coder/websocket` defaults). +- `BACKUP_AGENT_KEY` is never logged (`slog` ReplaceAttr redacts known keys + defensively; the value is also `json:"-"` in `Config`). +- State at rest is AES-256-GCM keyed by HKDF-SHA256 of the agent key. +- Docker socket is mounted read-only. +- Container runs as uid 65532 (distroless `nonroot`). +- Image is distroless static — no shell, no package manager, no busybox. + +## License + +MIT — see top-level `LICENSE` once added. diff --git a/apps/agent/cmd/agent/dumpstate.go b/apps/agent/cmd/agent/dumpstate.go new file mode 100644 index 0000000..1f70c19 --- /dev/null +++ b/apps/agent/cmd/agent/dumpstate.go @@ -0,0 +1,98 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/backupy/backupy/apps/agent/internal/config" + "github.com/backupy/backupy/apps/agent/internal/state" +) + +// stateDump is the pretty-printable shape of the agent's state file. +// Binary blobs are hex-encoded so the JSON stays UTF-8 clean. +type stateDump struct { + Path string `json:"path"` + SessionID string `json:"session_id,omitempty"` + ConfigVersion uint64 `json:"config_version"` + ConfigBytes string `json:"config_bytes_hex,omitempty"` + QueueDepth int `json:"queue_depth"` + Queue []dumpedJob `json:"queue,omitempty"` + LastHeartbeat int64 `json:"last_heartbeat_ms"` + Note string `json:"note,omitempty"` + Meta map[string]any `json:"meta,omitempty"` +} + +type dumpedJob struct { + RunID string `json:"run_id"` + PayloadHex string `json:"payload_hex"` +} + +func newDumpStateCmd() *cobra.Command { + var allowSecrets bool + cmd := &cobra.Command{ + Use: "dump-state", + Short: "Debug: print state.db contents as JSON", + Long: `Dump the contents of /var/lib/backup-agent/state.db as JSON. + +Refuses to print decrypted config or job payloads unless --allow-secrets +is passed, since those may include S3 presigned URLs or other sensitive +material received from the server.`, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + return err + } + s, err := state.Open(cfg.StateDBPath(), state.Options{AgentKey: cfg.AgentKey}) + if err != nil { + return err + } + defer s.Close() + + dump := stateDump{Path: cfg.StateDBPath()} + + if v, raw, err := s.LoadConfig(); err == nil { + dump.ConfigVersion = v + if allowSecrets { + dump.ConfigBytes = hex.EncodeToString(raw) + } else { + dump.Note = "config bytes omitted; re-run with --allow-secrets to include" + } + } + + if sid, err := s.LoadSession(); err == nil { + dump.SessionID = sid + } + if hb, err := s.LastHeartbeat(); err == nil { + dump.LastHeartbeat = hb + } + if d, err := s.QueueDepth(); err == nil { + dump.QueueDepth = d + } + + if allowSecrets { + if jobs, err := s.DequeueJobs(100); err == nil { + for _, j := range jobs { + dump.Queue = append(dump.Queue, dumpedJob{ + RunID: j.RunID, + PayloadHex: hex.EncodeToString(j.Payload), + }) + } + } + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(dump); err != nil { + return fmt.Errorf("encode: %w", err) + } + return nil + }, + } + cmd.Flags().BoolVar(&allowSecrets, "allow-secrets", false, + "include decrypted config and job payloads (may leak secrets)") + return cmd +} diff --git a/apps/agent/cmd/agent/healthcheck.go b/apps/agent/cmd/agent/healthcheck.go new file mode 100644 index 0000000..233fa8e --- /dev/null +++ b/apps/agent/cmd/agent/healthcheck.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/backupy/backupy/apps/agent/internal/config" + "github.com/backupy/backupy/apps/agent/internal/state" +) + +// newHealthCheckCmd implements the binary used by Docker's HEALTHCHECK +// instruction. We deliberately keep this command's surface tiny so the +// distroless image (no shell) can run it directly. +// +// Health criteria: +// +// 1. Required env vars are set (BACKUP_AGENT_KEY / BACKUP_SERVER_URL). +// 2. The state.db file can be opened (validates encryption key + on-disk +// integrity). +// +// In a future iteration we may extend this to query an internal UNIX +// socket served by `agent run` so connection liveness is also reflected. +func newHealthCheckCmd() *cobra.Command { + return &cobra.Command{ + Use: "health-check", + Short: "Probe used by Docker HEALTHCHECK; exits 0 when healthy", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load() + if err != nil { + fmt.Fprintln(os.Stderr, "health-check: config:", err) + os.Exit(1) + } + // Try to open the state DB — read+write lock validates that + // the file isn't corrupt and the key derivation still works. + s, err := state.Open(cfg.StateDBPath(), state.Options{AgentKey: cfg.AgentKey}) + if err != nil { + fmt.Fprintln(os.Stderr, "health-check: state:", err) + os.Exit(1) + } + _ = s.Close() + fmt.Println("ok") + return nil + }, + } +} diff --git a/apps/agent/cmd/agent/main.go b/apps/agent/cmd/agent/main.go new file mode 100644 index 0000000..8106928 --- /dev/null +++ b/apps/agent/cmd/agent/main.go @@ -0,0 +1,72 @@ +// Command agent is the Backupy agent — connects to the control plane, +// runs backups, and reports status. See docs/03-agent-spec.md. +// +// Subcommands (per spec §3.7): +// +// agent run — start the service loop (default) +// agent version — print build metadata +// agent health-check — used as Docker HEALTHCHECK; exits 0 when healthy +// agent dump-state — debug-only state dump as JSON +// +// `agent self-update` is documented in the spec but landing in task D-15. +package main + +import ( + "fmt" + "log/slog" + "os" + + "github.com/spf13/cobra" + + "github.com/backupy/backupy/apps/agent/internal/version" +) + +func main() { + root := newRootCmd() + if err := root.Execute(); err != nil { + // Cobra has already printed the error; this final line guarantees + // a non-zero exit code for environments that ignore Execute's err. + fmt.Fprintln(os.Stderr, "agent: fatal:", err) + os.Exit(1) + } +} + +// newRootCmd wires up all subcommands. Exported as a factory so tests +// (and future `agent_test.go`) can construct an isolated root command. +func newRootCmd() *cobra.Command { + root := &cobra.Command{ + Use: "agent", + Short: "Backupy agent — connects to the platform and runs backups", + Long: "Backupy agent service. See docs/03-agent-spec.md for the full spec.", + Version: version.Full(), + SilenceUsage: true, + SilenceErrors: false, + } + + // When invoked with no subcommand, default to `run`. This makes the + // Docker ENTRYPOINT clean: ["/usr/local/bin/agent"] just works. + root.AddCommand(newRunCmd()) + root.AddCommand(newVersionCmd()) + root.AddCommand(newHealthCheckCmd()) + root.AddCommand(newDumpStateCmd()) + + // Default to `run` when no args given (Docker CMD convention). + root.RunE = func(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + return fmt.Errorf("unknown command %q", args[0]) + } + return newRunCmd().RunE(cmd, args) + } + + return root +} + +// fatal logs at error level and exits non-zero. Used by subcommands that +// want a uniform exit path after a structured log line. +func fatal(logger *slog.Logger, msg string, err error) { + if logger == nil { + logger = slog.Default() + } + logger.Error(msg, slog.Any("err", err)) + os.Exit(1) +} diff --git a/apps/agent/cmd/agent/run.go b/apps/agent/cmd/agent/run.go new file mode 100644 index 0000000..544ed4e --- /dev/null +++ b/apps/agent/cmd/agent/run.go @@ -0,0 +1,301 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/backupy/backupy/apps/agent/internal/config" + "github.com/backupy/backupy/apps/agent/internal/discovery" + "github.com/backupy/backupy/apps/agent/internal/logging" + "github.com/backupy/backupy/apps/agent/internal/metrics" + "github.com/backupy/backupy/apps/agent/internal/pipeline" + agentproto "github.com/backupy/backupy/apps/agent/internal/proto" + "github.com/backupy/backupy/apps/agent/internal/queue" + "github.com/backupy/backupy/apps/agent/internal/state" + "github.com/backupy/backupy/apps/agent/internal/version" + "github.com/backupy/backupy/apps/agent/internal/wss" + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// discoveryInterval is the period for the auto-rescan loop. The spec +// pins this to "once an hour" — see docs/03-agent-spec.md. +const discoveryInterval = time.Hour + +func newRunCmd() *cobra.Command { + return &cobra.Command{ + Use: "run", + Short: "Start the agent service loop", + Long: `Run loads bootstrap config from env, opens the persistent state DB, +opens the WSS connection to the control plane, and blocks until SIGINT or +SIGTERM is received.`, + RunE: func(cmd *cobra.Command, args []string) error { + return runAgent() + }, + } +} + +func runAgent() error { + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("config: %w", err) + } + + logger := logging.New(cfg.LogLevel) + slog.SetDefault(logger) + + logger.Info("backupy agent starting", + slog.String("version", version.Version), + slog.String("commit", version.Commit), + slog.String("server_url", cfg.ServerURL), + slog.String("state_dir", cfg.StateDir), + ) + + // Persistent state — required to even start. + store, err := state.Open(cfg.StateDBPath(), state.Options{AgentKey: cfg.AgentKey}) + if err != nil { + return fmt.Errorf("state: %w", err) + } + defer func() { + if cerr := store.Close(); cerr != nil { + logger.Warn("state: close error", slog.Any("err", cerr)) + } + }() + + q := queue.NewBolt(store) + if depth, derr := q.Depth(); derr == nil && depth > 0 { + logger.Info("recovered pending jobs from queue", slog.Int("depth", depth)) + } + + // Auto-discovery + backup pipeline runner. + scanner := discovery.NewScanner(discovery.Config{ + DockerSocket: cfg.DockerSocket, + Logger: logger, + }) + + // In-memory AgentConfig snapshot — the WSS client pushes a fresh + // ConfigUpdate at register-time and on every change, and the + // pipeline runner needs the current Targets/Jobs to resolve a + // RunBackup.job_id to a Connection spec. + snap := newConfigSnapshot() + + runner := pipeline.NewRunner( + map[string]pipeline.Driver{ + "postgresql": pipeline.NewPgDump(), + "mysql": pipeline.NewMysqldump(), + "mariadb": pipeline.NewMysqldump(), + // --- B14 BEGIN + "mongodb": pipeline.NewMongoDump(), + "redis": pipeline.NewRedisDriver(), + "sqlite": pipeline.NewSqliteDriver(), + // --- B14 END + }, + pipeline.NewUploader(), + pipeline.WithLogger(logger), + pipeline.WithTargetLookup(snap), + pipeline.WithJobLookup(snap), + ) + + // The handlers struct is supplied by the WSS agent side; we plug + // the runner into OnRunBackup so an inbound RunBackup envelope + // drives an end-to-end backup. + var client *wss.Client // forward-declared for handler closures + handlers := &wss.Handlers{ + OnConfigUpdate: func(_ context.Context, msg *agentproto.ConfigUpdate) error { + c := msg.GetConfig() + if c == nil { + return nil + } + snap.Set(c) + logger.Info("wss: config update applied", + slog.Uint64("version", c.Version), + slog.Int("targets", len(c.Targets)), + slog.Int("jobs", len(c.Jobs))) + return nil + }, + OnRunBackup: func(ctx context.Context, msg *agentproto.RunBackup) error { + // Execute the backup pipeline in a goroutine so the read + // loop is not blocked while a multi-minute upload runs. + jobID := msg.JobId + runID := msg.RunId + go func() { + completed, runErr := runner.Run(ctx, msg) + if runErr != nil { + logger.Error("pipeline: run failed", + slog.String("job_id", jobID), + slog.String("run_id", runID), + slog.Any("err", runErr)) + if client != nil { + _ = client.Send(buildJobFailed(jobID, runID, runErr)) + } + return + } + if client != nil { + _ = client.Send(buildBackupCompleted(completed)) + } + }() + return nil + }, + OnCancelJob: func(_ context.Context, msg *agentproto.CancelJob) error { + logger.Info("wss: cancel job", slog.String("job_id", msg.JobId)) + return nil + }, + OnRunHealthCheck: func(_ context.Context, msg *agentproto.RunHealthCheck) error { + logger.Info("wss: run health check", slog.String("check_id", msg.CheckId)) + return nil + }, + OnSelfUpdate: func(_ context.Context, msg *agentproto.SelfUpdate) error { + logger.Info("wss: self-update requested", + slog.String("target_version", msg.TargetVersion)) + return nil + }, + } + + client = wss.NewClient(wss.Config{ + ServerURL: cfg.ServerURL, + AgentKey: cfg.AgentKey, + AgentVersion: version.Version, + AllowInsecure: cfg.DevAllowInsecure, + Capabilities: []string{"pg_dump", "mysqldump", "docker_discovery"}, + }, store, q, handlers, nil, logger) + + // Graceful shutdown on SIGINT/SIGTERM. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // --- D-19 BEGIN: Prometheus metrics endpoint (loopback by default). + metrics.SetBuildInfo(version.Version, version.Commit) + metrics.SetWSSState("disconnected") + if cfg.MetricsListenAddr != "" { + go func() { + if err := metrics.ListenAndServe(ctx, cfg.MetricsListenAddr); err != nil { + logger.Error("metrics: server error", + slog.String("addr", cfg.MetricsListenAddr), + slog.Any("err", err)) + } + }() + logger.Info("metrics endpoint enabled", + slog.String("addr", cfg.MetricsListenAddr)) + } + // --- D-19 END + + // Periodic discovery loop. Publishes DiscoveryReport envelopes + // through the WSS client once it is connected; while disconnected, + // Send buffers to the persistent queue automatically. + go runDiscoveryLoop(ctx, scanner, client, logger) + + if err := client.Start(ctx); err != nil { + return fmt.Errorf("wss: %w", err) + } + logger.Info("backupy agent stopped cleanly") + return nil +} + +// runDiscoveryLoop runs an immediate scan plus a periodic rescan every +// discoveryInterval, publishing DiscoveryReport envelopes through the +// supplied client. +func runDiscoveryLoop(ctx context.Context, scanner discovery.Scanner, client *wss.Client, logger *slog.Logger) { + scan := func() { + scanCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + containers, err := scanner.Scan(scanCtx) + if err != nil { + logger.Warn("discovery: scan failed", slog.Any("err", err)) + return + } + logger.Info("discovery: scan complete", slog.Int("containers", len(containers))) + report := discovery.BuildReport(containers) + if client == nil { + return + } + env := agentproto.NewEnvelope() + env.Payload = &backupv1.Envelope_Discovery{Discovery: report} + if err := client.Send(env); err != nil { + logger.Warn("discovery: send report", slog.Any("err", err)) + } + } + + scan() // run once on startup + ticker := time.NewTicker(discoveryInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + scan() + } + } +} + +// buildBackupCompleted wraps a BackupCompleted in an outbound envelope. +func buildBackupCompleted(c *backupv1.BackupCompleted) *agentproto.Envelope { + env := agentproto.NewEnvelope() + env.Payload = &backupv1.Envelope_BackupCompleted{BackupCompleted: c} + return env +} + +// buildJobFailed wraps a FAILED JobUpdate in an outbound envelope. +// Both job_id and run_id are populated so the scheduler can correlate +// the failure to its BackupRun row and apply the retry policy. +func buildJobFailed(jobID, runID string, runErr error) *agentproto.Envelope { + env := agentproto.NewEnvelope() + env.Payload = &backupv1.Envelope_JobUpdate{ + JobUpdate: &backupv1.JobUpdate{ + JobId: jobID, + RunId: runID, + Status: backupv1.JobStatus_FAILED, + ErrorMessage: runErr.Error(), + }, + } + return env +} + +// configSnapshot is the in-process projection of the latest AgentConfig +// the agent has received. It satisfies both pipeline.TargetLookup and +// pipeline.JobLookup so the runner can resolve job_id -> connection. +type configSnapshot struct { + mu sync.RWMutex + targets map[string]*backupv1.Target + jobs map[string]*backupv1.BackupJobSpec +} + +func newConfigSnapshot() *configSnapshot { + return &configSnapshot{ + targets: map[string]*backupv1.Target{}, + jobs: map[string]*backupv1.BackupJobSpec{}, + } +} + +func (s *configSnapshot) Set(cfg *backupv1.AgentConfig) { + s.mu.Lock() + defer s.mu.Unlock() + s.targets = map[string]*backupv1.Target{} + for _, t := range cfg.Targets { + s.targets[t.Id] = t + } + s.jobs = map[string]*backupv1.BackupJobSpec{} + for _, j := range cfg.Jobs { + s.jobs[j.Id] = j + } +} + +func (s *configSnapshot) Target(id string) (*backupv1.Target, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + t, ok := s.targets[id] + return t, ok +} + +func (s *configSnapshot) Job(id string) (*backupv1.BackupJobSpec, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + j, ok := s.jobs[id] + return j, ok +} diff --git a/apps/agent/cmd/agent/version.go b/apps/agent/cmd/agent/version.go new file mode 100644 index 0000000..4f977c7 --- /dev/null +++ b/apps/agent/cmd/agent/version.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/backupy/backupy/apps/agent/internal/version" +) + +func newVersionCmd() *cobra.Command { + var asJSON bool + cmd := &cobra.Command{ + Use: "version", + Short: "Print version, commit and build date", + RunE: func(cmd *cobra.Command, args []string) error { + info := version.Current() + if asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(info) + } + fmt.Printf("backupy-agent %s\n", version.Full()) + return nil + }, + } + cmd.Flags().BoolVar(&asJSON, "json", false, "emit version info as JSON") + return cmd +} diff --git a/apps/agent/go.mod b/apps/agent/go.mod new file mode 100644 index 0000000..fbd6a05 --- /dev/null +++ b/apps/agent/go.mod @@ -0,0 +1,38 @@ +module github.com/backupy/backupy/apps/agent + +go 1.23.0 + +require ( + github.com/backupy/backupy/packages/proto/gen/go/backupv1 v0.0.0-00010101000000-000000000000 + github.com/caarlos0/env/v11 v11.2.2 + github.com/coder/websocket v1.8.12 + github.com/klauspost/compress v1.18.0 + github.com/prometheus/client_golang v1.23.2 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.11.1 + go.etcd.io/bbolt v1.3.11 + golang.org/x/crypto v0.27.0 + google.golang.org/protobuf v1.36.8 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/sys v0.35.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +// The protobuf bindings are generated from packages/proto via `make proto` +// from the repo root. The replace directive lets agent code compile against +// the local generated sources during development. +replace github.com/backupy/backupy/packages/proto/gen/go/backupv1 => ../../packages/proto/gen/go/backupv1 diff --git a/apps/agent/go.sum b/apps/agent/go.sum new file mode 100644 index 0000000..c55c30f --- /dev/null +++ b/apps/agent/go.sum @@ -0,0 +1,64 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= +github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/agent/internal/config/config.go b/apps/agent/internal/config/config.go new file mode 100644 index 0000000..625db1e --- /dev/null +++ b/apps/agent/internal/config/config.go @@ -0,0 +1,125 @@ +// Package config loads agent configuration from environment variables. +// +// The bootstrap surface is intentionally tiny — per docs/03-agent-spec.md +// the only required inputs are the server URL and the agent key. Everything +// else (targets, schedules, S3 creds, etc.) arrives from the server via +// ConfigUpdate after the WSS handshake. +// +// Secrets are tagged `json:"-"` so they never leak through structured +// logging or `agent dump-state` output. +package config + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + + "github.com/caarlos0/env/v11" +) + +// agentKeyPattern enforces the documented BACKUP_AGENT_KEY format +// `bkpy_(live|test)_<32 base62 chars>`. The server issues keys in +// this exact shape — see docs/03-agent-spec.md and server task A-09. +var agentKeyPattern = regexp.MustCompile(`^bkpy_(live|test)_[A-Za-z0-9]{32}$`) + +// Config holds all agent bootstrap configuration. +type Config struct { + ServerURL string `env:"BACKUP_SERVER_URL,required" envDefault:"https://api.backupy.ru"` + AgentKey string `env:"BACKUP_AGENT_KEY,required" json:"-"` + StateDir string `env:"BACKUP_STATE_DIR" envDefault:"/var/lib/backup-agent"` + LogLevel string `env:"BACKUP_LOG_LEVEL" envDefault:"info"` + DockerSocket string `env:"BACKUP_DOCKER_SOCKET" envDefault:"/var/run/docker.sock"` + + // DevAllowInsecure relaxes the https:// requirement on ServerURL. + // Intended for local development against a plaintext server only. + DevAllowInsecure bool `env:"BACKUP_DEV_ALLOW_INSECURE" envDefault:"false"` + + // MetricsListenAddr is the bind address for the Prometheus + // `/metrics` endpoint (D-19). Default is loopback only — + // 127.0.0.1:9090. Set to empty to disable the metrics server. + // SECURITY: never bind to 0.0.0.0 in production; the endpoint + // reveals job IDs and run cadence usable for host fingerprinting. + MetricsListenAddr string `env:"BACKUP_METRICS_LISTEN_ADDR" envDefault:"127.0.0.1:9090"` +} + +// Load parses environment variables into a Config and validates them. +// Missing required vars or malformed values cause Load to return an error +// describing the problem; nothing about secret values is included. +func Load() (*Config, error) { + cfg := &Config{} + if err := env.Parse(cfg); err != nil { + return nil, fmt.Errorf("config: parse env: %w", err) + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return cfg, nil +} + +// Validate enforces the documented constraints on each field. +// +// - ServerURL must parse as an https:// URL (http:// only with +// BACKUP_DEV_ALLOW_INSECURE=true). +// - AgentKey must match the canonical `bkpy_(live|test)_…` pattern. +// - StateDir must be writable; we test by creating and removing a temp +// file so a misconfigured volume mount fails fast at startup. +func (c *Config) Validate() error { + if err := validateServerURL(c.ServerURL, c.DevAllowInsecure); err != nil { + return err + } + if !agentKeyPattern.MatchString(c.AgentKey) { + return errors.New("config: BACKUP_AGENT_KEY has invalid format; expected bkpy_(live|test)_<32 alnum>") + } + if err := validateStateDirWritable(c.StateDir); err != nil { + return err + } + return nil +} + +func validateServerURL(raw string, allowInsecure bool) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("config: BACKUP_SERVER_URL is not a valid URL: %w", err) + } + if u.Host == "" { + return errors.New("config: BACKUP_SERVER_URL is missing host") + } + switch u.Scheme { + case "https": + return nil + case "http": + if !allowInsecure { + return errors.New("config: BACKUP_SERVER_URL must use https:// (set BACKUP_DEV_ALLOW_INSECURE=true for local dev)") + } + return nil + default: + return fmt.Errorf("config: BACKUP_SERVER_URL has unsupported scheme %q (expected https)", u.Scheme) + } +} + +func validateStateDirWritable(dir string) error { + if dir == "" { + return errors.New("config: BACKUP_STATE_DIR must not be empty") + } + // Ensure the directory exists; create it (and parents) if missing. + // 0o700 — only the agent UID should ever touch state. + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("config: cannot create state dir %q: %w", dir, err) + } + probe, err := os.CreateTemp(dir, ".writable-probe-*") + if err != nil { + return fmt.Errorf("config: state dir %q is not writable: %w", dir, err) + } + name := probe.Name() + _ = probe.Close() + _ = os.Remove(name) + return nil +} + +// StateDBPath returns the absolute path to the BoltDB state file. +func (c *Config) StateDBPath() string { + return filepath.Join(c.StateDir, "state.db") +} diff --git a/apps/agent/internal/config/config_test.go b/apps/agent/internal/config/config_test.go new file mode 100644 index 0000000..430e236 --- /dev/null +++ b/apps/agent/internal/config/config_test.go @@ -0,0 +1,107 @@ +package config + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const validKey = "bkpy_live_abcdefghijklmnopqrstuvwxyz012345" + +func TestValidate_Happy(t *testing.T) { + dir := t.TempDir() + cfg := &Config{ + ServerURL: "https://api.backupy.ru", + AgentKey: validKey, + StateDir: dir, + LogLevel: "info", + } + require.NoError(t, cfg.Validate()) + require.Equal(t, filepath.Join(dir, "state.db"), cfg.StateDBPath()) +} + +func TestValidate_ServerURL(t *testing.T) { + cases := []struct { + name string + url string + insec bool + errSub string + }{ + {"https ok", "https://api.backupy.ru", false, ""}, + {"http rejected by default", "http://localhost:8080", false, "must use https"}, + {"http allowed with dev flag", "http://localhost:8080", true, ""}, + {"ftp rejected", "ftp://example.com", false, "unsupported scheme"}, + {"missing host", "https://", false, "missing host"}, + {"unparsable", "://broken", false, "valid URL"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateServerURL(tc.url, tc.insec) + if tc.errSub == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + require.Contains(t, err.Error(), tc.errSub) + }) + } +} + +func TestValidate_AgentKey(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + key string + ok bool + }{ + {"live key", "bkpy_live_abcdefghijklmnopqrstuvwxyz012345", true}, + {"test key", "bkpy_test_abcdefghijklmnopqrstuvwxyz012345", true}, + {"wrong prefix", "bkpy_dev_abcdefghijklmnopqrstuvwxyz012345", false}, + {"too short", "bkpy_live_abc", false}, + {"too long", "bkpy_live_abcdefghijklmnopqrstuvwxyz0123456", false}, + {"invalid char", "bkpy_live_abcdefghijklmnopqrstuvwxyz01234!", false}, + {"empty", "", false}, + {"missing underscore", "bkpylive_abcdefghijklmnopqrstuvwxyz012345", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &Config{ + ServerURL: "https://api.backupy.ru", + AgentKey: tc.key, + StateDir: dir, + } + err := cfg.Validate() + if tc.ok { + require.NoError(t, err) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), "BACKUP_AGENT_KEY") + } + }) + } +} + +func TestValidate_StateDir(t *testing.T) { + t.Run("empty rejected", func(t *testing.T) { + err := validateStateDirWritable("") + require.Error(t, err) + }) + t.Run("creates missing dir", func(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "nested", "state") + require.NoError(t, validateStateDirWritable(nested)) + }) + t.Run("redaction in errors", func(t *testing.T) { + // Sanity check: error text never mentions the agent key. + cfg := &Config{ + ServerURL: "https://api.backupy.ru", + AgentKey: validKey, + StateDir: "", // invalid + } + err := cfg.Validate() + require.Error(t, err) + require.False(t, strings.Contains(err.Error(), validKey)) + }) +} diff --git a/apps/agent/internal/discovery/discovery.go b/apps/agent/internal/discovery/discovery.go new file mode 100644 index 0000000..84073bd --- /dev/null +++ b/apps/agent/internal/discovery/discovery.go @@ -0,0 +1,21 @@ +// Package discovery scans the local Docker socket and reports discovered +// database containers (postgres, mysql, mariadb, mongo, redis). +// +// Implementation notes (D-05): +// +// - We talk to /var/run/docker.sock directly over HTTP via a custom +// net.Dialer wired into http.Transport. NO Docker SDK dependency. +// - Container detection is purely image-name heuristic plus a small set +// of env-var name hints. We do NOT exfiltrate env values, only the +// keys — the spec is explicit that plaintext secrets must stay on the +// host. See docs/03-agent-spec.md → "Auto-discovery". +// - The returned Container struct mirrors the proto DiscoveredContainer +// message so callers can map 1:1 without an extra translation layer. +package discovery + +// File split: +// - scanner.go : the Scanner interface, Container/PortBinding types, +// NewDockerScanner constructor, BuildReport helper. +// - docker.go : the unix-socket HTTP client + Docker API parsing logic. +// +// This file exists so go-doc on the package shows the high-level overview. diff --git a/apps/agent/internal/discovery/docker.go b/apps/agent/internal/discovery/docker.go new file mode 100644 index 0000000..c1d60ae --- /dev/null +++ b/apps/agent/internal/discovery/docker.go @@ -0,0 +1,428 @@ +package discovery + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// dockerAPIVersion is the minimum Docker Engine API version the agent +// negotiates with. /v1.41/ corresponds to Engine 20.10 (released 2020-12) +// — every supported host distribution ships at least this version. +const dockerAPIVersion = "v1.41" + +// dockerHTTPTimeout caps a single Docker API call. The full Scan calls +// (1 list + N inspects), so we keep each individual request snappy. +const dockerHTTPTimeout = 5 * time.Second + +// dbTypeByImagePrefix maps a normalised image basename prefix to the +// DetectedDBType string the rest of the agent uses. Order matters only +// for documentation; lookup is exact-prefix. +var dbTypeByImagePrefix = []struct { + prefix string + dbType string +}{ + // Postgres official + common forks. + {"postgres", "postgresql"}, + {"postgis/postgis", "postgresql"}, + {"timescale/timescaledb", "postgresql"}, + {"bitnami/postgresql", "postgresql"}, + // MySQL (server, percona). Order: mariadb BEFORE mysql so that the + // `mariadb` image is not swallowed by a `mysql` substring rule. + {"mariadb", "mariadb"}, + {"bitnami/mariadb", "mariadb"}, + {"mysql", "mysql"}, + {"percona", "mysql"}, + {"bitnami/mysql", "mysql"}, + // MongoDB. + {"mongo", "mongodb"}, + {"bitnami/mongodb", "mongodb"}, + // Redis. + {"redis", "redis"}, + {"bitnami/redis", "redis"}, +} + +// envHintKeysByDBType lists which env-var KEYS are exposed in DiscoveryReport. +// Values stay on host. The intent is to populate the connection form in the +// UI: "this container has POSTGRES_USER set — do you want to pre-fill?". +var envHintKeysByDBType = map[string][]string{ + "postgresql": {"POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_DB", "POSTGRESQL_USER", "POSTGRESQL_PASSWORD", "POSTGRESQL_DATABASE", "PGUSER", "PGPASSWORD", "PGDATABASE"}, + "mysql": {"MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_ROOT_PASSWORD", "MYSQL_DATABASE", "MYSQL_ALLOW_EMPTY_PASSWORD"}, + "mariadb": {"MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_ROOT_PASSWORD", "MYSQL_DATABASE", "MARIADB_USER", "MARIADB_PASSWORD", "MARIADB_ROOT_PASSWORD", "MARIADB_DATABASE"}, + "mongodb": {"MONGO_INITDB_ROOT_USERNAME", "MONGO_INITDB_ROOT_PASSWORD", "MONGO_INITDB_DATABASE"}, + "redis": {"REDIS_PASSWORD", "REDIS_USERNAME"}, +} + +// envHintAllowSet collapses envHintKeysByDBType into a single membership +// set used by filterEnv — every key in this set is allowed across all +// containers, but the value is replaced with "set" sentinel. +var envHintAllowSet = func() map[string]struct{} { + out := make(map[string]struct{}) + for _, ks := range envHintKeysByDBType { + for _, k := range ks { + out[k] = struct{}{} + } + } + return out +}() + +// dockerScanner is the production Scanner implementation. +type dockerScanner struct { + httpClient *http.Client + baseURL string // e.g. "http://docker/v1.41" + logger *slog.Logger +} + +func newDockerScanner(socketPath string, logger *slog.Logger) *dockerScanner { + if logger == nil { + logger = slog.Default() + } + if !strings.HasPrefix(socketPath, "http://") && !strings.HasPrefix(socketPath, "https://") { + // Treat the path as a unix socket. The HTTP client below dials + // the file regardless of the host portion of the URL. + return &dockerScanner{ + httpClient: newUnixHTTPClient(socketPath), + baseURL: "http://docker/" + dockerAPIVersion, + logger: logger.With(slog.String("component", "discovery")), + } + } + // Test path: socketPath is an http(s):// base URL pointing at a fake + // daemon. We append the API version segment so production and tests + // hit identical relative paths. + base := strings.TrimRight(socketPath, "/") + return &dockerScanner{ + httpClient: &http.Client{Timeout: dockerHTTPTimeout}, + baseURL: base + "/" + dockerAPIVersion, + logger: logger.With(slog.String("component", "discovery")), + } +} + +// newUnixHTTPClient builds an *http.Client whose transport dials the +// given unix socket on every request. The URL host segment is ignored. +func newUnixHTTPClient(socket string) *http.Client { + dialer := &net.Dialer{Timeout: 2 * time.Second} + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, "unix", socket) + }, + // Keep the connection pool tiny — discovery runs at most every + // hour and we don't want lingering connections to a host file. + MaxIdleConns: 2, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + TLSHandshakeTimeout: time.Second, + } + return &http.Client{Transport: transport, Timeout: dockerHTTPTimeout} +} + +// dockerListEntry is a partial mirror of the /containers/json response. +// Only the fields we actually need are decoded — the Docker API ships +// dozens of fields we don't care about. +type dockerListEntry struct { + ID string `json:"Id"` + Names []string + Image string + State string + Ports []struct { + PrivatePort uint32 `json:"PrivatePort"` + PublicPort uint32 `json:"PublicPort"` + Type string `json:"Type"` + } + NetworkSettings struct { + Networks map[string]json.RawMessage `json:"Networks"` + } `json:"NetworkSettings"` +} + +// dockerInspectResponse mirrors the small subset of /containers/{id}/json +// we need: container config (env, image) and network settings. +type dockerInspectResponse struct { + ID string `json:"Id"` + Name string `json:"Name"` + State struct { + Running bool `json:"Running"` + } `json:"State"` + Config struct { + Image string `json:"Image"` + Env []string `json:"Env"` + } `json:"Config"` + HostConfig struct { + NetworkMode string `json:"NetworkMode"` + } `json:"HostConfig"` + NetworkSettings struct { + Networks map[string]json.RawMessage `json:"Networks"` + Ports map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` + } `json:"Ports"` + } `json:"NetworkSettings"` +} + +// Scan implements Scanner.Scan. +func (s *dockerScanner) Scan(ctx context.Context) ([]DiscoveredContainer, error) { + list, err := s.listContainers(ctx) + if err != nil { + return nil, fmt.Errorf("discovery: list containers: %w", err) + } + out := make([]DiscoveredContainer, 0, len(list)) + for _, c := range list { + dbType := detectDBType(c.Image) + if dbType == "" { + continue + } + details, err := s.inspectContainer(ctx, c.ID) + if err != nil { + s.logger.Warn("discovery: inspect failed", slog.String("container_id", c.ID), slog.Any("err", err)) + continue + } + // Only running containers are useful for live discovery. + if !details.State.Running { + continue + } + out = append(out, buildContainer(c, details, dbType)) + } + return out, nil +} + +// listContainers issues GET /containers/json (running containers only). +func (s *dockerScanner) listContainers(ctx context.Context) ([]dockerListEntry, error) { + endpoint := s.baseURL + "/containers/json" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("build list request: %w", err) + } + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("docker GET %s: %w", endpoint, err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("docker list containers: HTTP %d", resp.StatusCode) + } + var entries []dockerListEntry + if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil { + return nil, fmt.Errorf("decode list response: %w", err) + } + return entries, nil +} + +// inspectContainer issues GET /containers/{id}/json. +func (s *dockerScanner) inspectContainer(ctx context.Context, id string) (*dockerInspectResponse, error) { + endpoint := s.baseURL + "/containers/" + url.PathEscape(id) + "/json" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("build inspect request: %w", err) + } + resp, err := s.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("docker GET %s: %w", endpoint, err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, errors.New("container not found") + } + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("docker inspect: HTTP %d", resp.StatusCode) + } + var details dockerInspectResponse + if err := json.NewDecoder(resp.Body).Decode(&details); err != nil { + return nil, fmt.Errorf("decode inspect response: %w", err) + } + return &details, nil +} + +// detectDBType applies the dbTypeByImagePrefix table. The image string +// may include a registry, tag, or digest — we strip those before +// matching so "ghcr.io/postgres:16" still resolves to "postgresql". +func detectDBType(image string) string { + norm := normaliseImage(image) + for _, rule := range dbTypeByImagePrefix { + if norm == rule.prefix || strings.HasPrefix(norm, rule.prefix+":") || strings.HasPrefix(norm, rule.prefix+"/") { + return rule.dbType + } + // also match "" when prefix is a bare word + // like "postgres" — e.g. "postgresql" image alias. + if !strings.Contains(rule.prefix, "/") && strings.HasPrefix(norm, rule.prefix) { + rest := norm[len(rule.prefix):] + if rest == "" || rest[0] == ':' || rest[0] == '/' || isAlnumExtension(rule.prefix, rest) { + return rule.dbType + } + } + } + return "" +} + +// isAlnumExtension allows a small whitelist of suffixes after a bare +// prefix — e.g. "postgresql" after "postgres", or "mysql8" tag-free +// build images. Conservative to avoid false positives like "mysqld-exporter". +func isAlnumExtension(prefix, rest string) bool { + switch prefix { + case "postgres": + return rest == "ql" + case "mysql": + // digits only, e.g. "mysql8". + for _, r := range rest { + if r < '0' || r > '9' { + return false + } + } + return rest != "" + } + return false +} + +// normaliseImage strips registry host, digest, and lowercases the result. +// It keeps the namespace ("bitnami/postgresql") because some rules match +// the namespaced form. +func normaliseImage(image string) string { + s := strings.ToLower(strings.TrimSpace(image)) + if at := strings.Index(s, "@"); at >= 0 { + s = s[:at] // drop digest + } + // Strip registry host iff the first segment contains a "." or ":" + // (port). Docker official images on Docker Hub have no host segment. + if slash := strings.Index(s, "/"); slash > 0 { + first := s[:slash] + if strings.ContainsAny(first, ".:") { + s = s[slash+1:] + } + } + return s +} + +// buildContainer projects raw Docker API structs into a DiscoveredContainer. +func buildContainer(list dockerListEntry, det *dockerInspectResponse, dbType string) DiscoveredContainer { + name := strings.TrimPrefix(det.Name, "/") + if name == "" && len(list.Names) > 0 { + name = strings.TrimPrefix(list.Names[0], "/") + } + + // Networks: prefer inspect response; fall back to list entry. + networks := make([]string, 0, len(det.NetworkSettings.Networks)) + for n := range det.NetworkSettings.Networks { + networks = append(networks, n) + } + if len(networks) == 0 { + for n := range list.NetworkSettings.Networks { + networks = append(networks, n) + } + } + + // Ports: build from the inspect response's NetworkSettings.Ports map + // which carries both exposed and published ports. The list entry's + // Ports field is also accepted as a fallback. + ports := portsFromInspect(det.NetworkSettings.Ports) + if len(ports) == 0 { + for _, p := range list.Ports { + ports = append(ports, PortBinding{ + ContainerPort: p.PrivatePort, + HostPort: p.PublicPort, + Protocol: defaultProto(p.Type), + }) + } + } + + return DiscoveredContainer{ + ContainerID: det.ID, + Name: name, + Image: det.Config.Image, + DetectedDBType: dbType, + Networks: networks, + EnvHints: filterEnv(det.Config.Env), + Ports: ports, + } +} + +// portsFromInspect parses the "Ports" map from /containers/{id}/json. +// The keys look like "5432/tcp"; the values are arrays of host bindings +// (one per host interface). A nil/empty bindings array means "exposed +// but not published" — HostPort stays 0. +func portsFromInspect(in map[string][]struct { + HostIP string `json:"HostIp"` + HostPort string `json:"HostPort"` +}) []PortBinding { + if len(in) == 0 { + return nil + } + out := make([]PortBinding, 0, len(in)) + for key, bindings := range in { + port, proto := parsePortKey(key) + if port == 0 { + continue + } + if len(bindings) == 0 { + out = append(out, PortBinding{ContainerPort: port, Protocol: proto}) + continue + } + for _, b := range bindings { + out = append(out, PortBinding{ + ContainerPort: port, + HostPort: parsePort(b.HostPort), + Protocol: proto, + }) + } + } + return out +} + +// parsePortKey splits "5432/tcp" → (5432, "tcp"). Defaults protocol to "tcp". +func parsePortKey(key string) (uint32, string) { + slash := strings.Index(key, "/") + if slash < 0 { + return parsePort(key), "tcp" + } + return parsePort(key[:slash]), defaultProto(key[slash+1:]) +} + +func parsePort(s string) uint32 { + if s == "" { + return 0 + } + var v uint32 + for _, r := range s { + if r < '0' || r > '9' { + return 0 + } + v = v*10 + uint32(r-'0') + if v > 65535 { + return 0 + } + } + return v +} + +func defaultProto(p string) string { + p = strings.ToLower(strings.TrimSpace(p)) + if p == "" { + return "tcp" + } + return p +} + +// filterEnv reads `KEY=VALUE` entries from the container env. For any +// key in the allow-set we emit the key with a sentinel value ("set"), so +// downstream code sees structural presence but never the secret. +// +// Returns an empty map (not nil) when no hints are found, so the proto +// `map` always serialises an empty map rather than nil. +func filterEnv(env []string) map[string]string { + out := make(map[string]string) + for _, e := range env { + eq := strings.IndexByte(e, '=') + if eq <= 0 { + continue + } + key := e[:eq] + if _, ok := envHintAllowSet[key]; !ok { + continue + } + out[key] = "set" + } + return out +} diff --git a/apps/agent/internal/discovery/docker_test.go b/apps/agent/internal/discovery/docker_test.go new file mode 100644 index 0000000..b636319 --- /dev/null +++ b/apps/agent/internal/discovery/docker_test.go @@ -0,0 +1,232 @@ +package discovery + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeDockerServer wires a single httptest.Server that pretends to be the +// Docker Engine HTTP API on the configured API-version prefix. +func fakeDockerServer(t *testing.T, list []map[string]any, inspect map[string]map[string]any) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + listPath := "/" + dockerAPIVersion + "/containers/json" + mux.HandleFunc(listPath, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(list) + }) + mux.HandleFunc("/"+dockerAPIVersion+"/containers/", func(w http.ResponseWriter, r *http.Request) { + // expect path: /vX/containers/{id}/json + trim := strings.TrimPrefix(r.URL.Path, "/"+dockerAPIVersion+"/containers/") + id := strings.TrimSuffix(trim, "/json") + if id == "" { + http.NotFound(w, r) + return + } + body, ok := inspect[id] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + }) + return httptest.NewServer(mux) +} + +func TestScan_DetectsPostgres(t *testing.T) { + srv := fakeDockerServer(t, + []map[string]any{{ + "Id": "abc", + "Names": []string{"/db"}, + "Image": "postgres:16", + "State": "running", + }}, + map[string]map[string]any{ + "abc": { + "Id": "abc", + "Name": "/db", + "State": map[string]any{"Running": true}, + "Config": map[string]any{ + "Image": "postgres:16", + "Env": []string{"POSTGRES_USER=app", "POSTGRES_PASSWORD=hunter2", "POSTGRES_DB=app", "PATH=/usr/bin"}, + }, + "NetworkSettings": map[string]any{ + "Networks": map[string]any{"bridge": map[string]any{}}, + "Ports": map[string]any{ + "5432/tcp": []map[string]any{{"HostIp": "0.0.0.0", "HostPort": "5432"}}, + }, + }, + }, + }, + ) + defer srv.Close() + + s := newDockerScanner(srv.URL, nil) + got, err := s.Scan(context.Background()) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "postgresql", got[0].DetectedDBType) + require.Equal(t, "db", got[0].Name) + require.Contains(t, got[0].EnvHints, "POSTGRES_PASSWORD") + require.NotContains(t, got[0].EnvHints, "PATH", "PATH should be filtered out") + // Critically: values must not leak — we replace with "set". + for _, v := range got[0].EnvHints { + require.Equal(t, "set", v, "env values must never appear in EnvHints") + } + require.Len(t, got[0].Ports, 1) + require.Equal(t, uint32(5432), got[0].Ports[0].ContainerPort) + require.Equal(t, uint32(5432), got[0].Ports[0].HostPort) + require.Equal(t, "tcp", got[0].Ports[0].Protocol) +} + +func TestScan_DetectsAllSupportedTypes(t *testing.T) { + cases := []struct { + image string + dbType string + }{ + {"postgres:16", "postgresql"}, + {"postgres", "postgresql"}, + {"timescale/timescaledb:latest-pg16", "postgresql"}, + {"mysql:8.0", "mysql"}, + {"percona:8", "mysql"}, + {"mariadb:11", "mariadb"}, + {"mongo:7", "mongodb"}, + {"redis:7-alpine", "redis"}, + } + list := make([]map[string]any, 0, len(cases)) + inspect := make(map[string]map[string]any, len(cases)) + for i, c := range cases { + id := "c" + string(rune('a'+i)) + list = append(list, map[string]any{ + "Id": id, "Names": []string{"/" + id}, "Image": c.image, "State": "running", + }) + inspect[id] = map[string]any{ + "Id": id, + "Name": "/" + id, + "State": map[string]any{"Running": true}, + "Config": map[string]any{ + "Image": c.image, + "Env": []string{}, + }, + } + } + srv := fakeDockerServer(t, list, inspect) + defer srv.Close() + + s := newDockerScanner(srv.URL, nil) + got, err := s.Scan(context.Background()) + require.NoError(t, err) + require.Len(t, got, len(cases)) + + gotByImage := make(map[string]string) + for _, c := range got { + gotByImage[c.Image] = c.DetectedDBType + } + for _, c := range cases { + require.Equal(t, c.dbType, gotByImage[c.image], "image %q", c.image) + } +} + +func TestScan_SkipsNonDBImages(t *testing.T) { + srv := fakeDockerServer(t, + []map[string]any{ + {"Id": "x", "Names": []string{"/web"}, "Image": "nginx:1.27", "State": "running"}, + {"Id": "y", "Names": []string{"/app"}, "Image": "myorg/api:1.2", "State": "running"}, + // mysqld-exporter must NOT be classified as mysql. + {"Id": "z", "Names": []string{"/exp"}, "Image": "prom/mysqld-exporter", "State": "running"}, + }, + map[string]map[string]any{}, + ) + defer srv.Close() + + s := newDockerScanner(srv.URL, nil) + got, err := s.Scan(context.Background()) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestScan_SkipsStoppedContainers(t *testing.T) { + srv := fakeDockerServer(t, + []map[string]any{{"Id": "abc", "Names": []string{"/db"}, "Image": "postgres:16", "State": "exited"}}, + map[string]map[string]any{ + "abc": { + "Id": "abc", + "Name": "/db", + "State": map[string]any{"Running": false}, + "Config": map[string]any{"Image": "postgres:16", "Env": []string{}}, + }, + }, + ) + defer srv.Close() + + s := newDockerScanner(srv.URL, nil) + got, err := s.Scan(context.Background()) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestScan_ContainerWithNoPublishedPorts(t *testing.T) { + srv := fakeDockerServer(t, + []map[string]any{{"Id": "abc", "Names": []string{"/db"}, "Image": "postgres:16", "State": "running"}}, + map[string]map[string]any{ + "abc": { + "Id": "abc", + "Name": "/db", + "State": map[string]any{"Running": true}, + "Config": map[string]any{ + "Image": "postgres:16", + "Env": []string{"POSTGRES_USER=app"}, + }, + "NetworkSettings": map[string]any{ + "Ports": map[string]any{ + // EXPOSE 5432 with no host binding. + "5432/tcp": nil, + }, + }, + }, + }, + ) + defer srv.Close() + + s := newDockerScanner(srv.URL, nil) + got, err := s.Scan(context.Background()) + require.NoError(t, err) + require.Len(t, got, 1) + require.Len(t, got[0].Ports, 1) + require.Equal(t, uint32(5432), got[0].Ports[0].ContainerPort) + require.Equal(t, uint32(0), got[0].Ports[0].HostPort, "exposed-but-not-published HostPort must be 0") +} + +func TestNormaliseImage(t *testing.T) { + cases := map[string]string{ + "postgres:16": "postgres:16", + "ghcr.io/example/postgres:16": "example/postgres:16", + "localhost:5000/mariadb:latest": "mariadb:latest", + "postgres@sha256:deadbeef": "postgres", + "BITNAMI/Postgresql:14": "bitnami/postgresql:14", + } + for in, want := range cases { + require.Equal(t, want, normaliseImage(in), "input %q", in) + } +} + +func TestBuildReport_NoValueLeakage(t *testing.T) { + report := BuildReport([]DiscoveredContainer{{ + ContainerID: "abc", + Name: "db", + Image: "postgres:16", + DetectedDBType: "postgresql", + EnvHints: map[string]string{"POSTGRES_USER": "set", "POSTGRES_PASSWORD": "set"}, + }}) + require.Len(t, report.Containers, 1) + for _, v := range report.Containers[0].EnvHints { + require.Equal(t, "set", v) + } +} diff --git a/apps/agent/internal/discovery/scanner.go b/apps/agent/internal/discovery/scanner.go new file mode 100644 index 0000000..2418d46 --- /dev/null +++ b/apps/agent/internal/discovery/scanner.go @@ -0,0 +1,110 @@ +package discovery + +import ( + "context" + "log/slog" + + agentproto "github.com/backupy/backupy/apps/agent/internal/proto" + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// DiscoveredContainer is the in-process representation of one discovered +// database candidate. Field names track the proto DiscoveredContainer +// message (see packages/proto/v1/agent_to_server.proto). +// +// Container is kept as an alias for backwards compatibility with the +// pre-D-05 skeleton — callers that already imported discovery.Container +// keep compiling. +type DiscoveredContainer struct { + ContainerID string + Name string + Image string + DetectedDBType string // "postgresql" | "mysql" | "mariadb" | "mongodb" | "redis" + Networks []string + EnvHints map[string]string // env-var KEYS only — values stay on host + Ports []PortBinding +} + +// Container is an alias retained for callers from the pre-D-05 skeleton. +// New code should use DiscoveredContainer. +type Container = DiscoveredContainer + +// PortBinding describes a single container port (optionally) published +// to the host. ContainerPort is always populated; HostPort is 0 when the +// port is exposed but not published. +type PortBinding struct { + ContainerPort uint32 + HostPort uint32 + Protocol string +} + +// Scanner runs container discovery against the Docker daemon. +type Scanner interface { + // Scan returns all currently-detected database containers. The + // returned slice is empty (not nil) when no containers match. + Scan(ctx context.Context) ([]DiscoveredContainer, error) +} + +// Config configures a Scanner constructed via NewScanner. Kept for +// backwards compatibility — new callers use NewDockerScanner directly. +type Config struct { + DockerSocket string + Logger *slog.Logger +} + +// NewScanner returns the default Docker-socket-backed Scanner using the +// given configuration. The logger is optional; if nil, slog.Default() is +// used. +func NewScanner(cfg Config) Scanner { + logger := cfg.Logger + if logger == nil { + logger = slog.Default() + } + socket := cfg.DockerSocket + if socket == "" { + socket = "/var/run/docker.sock" + } + return newDockerScanner(socket, logger) +} + +// NewDockerScanner returns a Scanner that talks to the Docker daemon at +// the unix socket path provided. Useful for unit tests that swap in a +// test HTTP server (see docker_test.go). +func NewDockerScanner(socketPath string) Scanner { + return newDockerScanner(socketPath, slog.Default()) +} + +// BuildReport projects a slice of discovered containers onto the proto +// DiscoveryReport message so the WSS client can send it without knowing +// about the Container struct. +func BuildReport(containers []DiscoveredContainer) *agentproto.DiscoveryReport { + report := &agentproto.DiscoveryReport{ + Containers: make([]*backupv1.DiscoveredContainer, 0, len(containers)), + } + for _, c := range containers { + ports := make([]*backupv1.PortBinding, 0, len(c.Ports)) + for _, p := range c.Ports { + ports = append(ports, &backupv1.PortBinding{ + ContainerPort: p.ContainerPort, + HostPort: p.HostPort, + Protocol: p.Protocol, + }) + } + // Defensive copy of env hints so subsequent mutation of the + // source map cannot leak into the serialised message. + hints := make(map[string]string, len(c.EnvHints)) + for k, v := range c.EnvHints { + hints[k] = v + } + report.Containers = append(report.Containers, &backupv1.DiscoveredContainer{ + ContainerId: c.ContainerID, + Name: c.Name, + Image: c.Image, + DetectedDbType: c.DetectedDBType, + Networks: append([]string(nil), c.Networks...), + EnvHints: hints, + Ports: ports, + }) + } + return report +} diff --git a/apps/agent/internal/logging/logging.go b/apps/agent/internal/logging/logging.go new file mode 100644 index 0000000..8ba6d9d --- /dev/null +++ b/apps/agent/internal/logging/logging.go @@ -0,0 +1,62 @@ +// Package logging configures the agent's structured logger built on +// the standard library's log/slog package. +// +// Output is always JSON on stdout (per docs/03-agent-spec.md — agent logs +// are eventually streamed to the server, so structured form is mandatory). +// The dev profile lowers verbosity by disabling source positions. +// +// BACKUP_AGENT_KEY is never logged — see config.Config which tags it +// `json:"-"` and the redactKey helper here for defence-in-depth. +package logging + +import ( + "io" + "log/slog" + "os" + "strings" +) + +// New returns a configured *slog.Logger writing JSON to stdout. +func New(level string) *slog.Logger { + return NewWithWriter(os.Stdout, level) +} + +// NewWithWriter is the same as New but writes to a caller-supplied io.Writer. +// Useful for tests. +func NewWithWriter(w io.Writer, level string) *slog.Logger { + opts := &slog.HandlerOptions{ + Level: parseLevel(level), + ReplaceAttr: redactSecrets, + } + return slog.New(slog.NewJSONHandler(w, opts)) +} + +// parseLevel converts a textual log level to slog.Level. Unknown levels +// default to info so a typo never silently silences the logger. +func parseLevel(s string) slog.Level { + switch strings.ToLower(strings.TrimSpace(s)) { + case "trace", "debug": + return slog.LevelDebug + case "info", "": + return slog.LevelInfo + case "warn", "warning": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} + +// redactSecrets is a slog.HandlerOptions.ReplaceAttr hook that masks any +// attribute named like a secret. Defence-in-depth — keys should never be +// passed into logs in the first place, but if a caller slips up the value +// is replaced before serialisation. +func redactSecrets(_ []string, a slog.Attr) slog.Attr { + key := strings.ToLower(a.Key) + switch key { + case "agent_key", "backup_agent_key", "password", "secret", "token", "authorization": + return slog.String(a.Key, "***") + } + return a +} diff --git a/apps/agent/internal/logging/logging_test.go b/apps/agent/internal/logging/logging_test.go new file mode 100644 index 0000000..61e5ac1 --- /dev/null +++ b/apps/agent/internal/logging/logging_test.go @@ -0,0 +1,54 @@ +package logging + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseLevel(t *testing.T) { + tests := []struct { + in string + want slog.Level + }{ + {"debug", slog.LevelDebug}, + {"trace", slog.LevelDebug}, + {"info", slog.LevelInfo}, + {"INFO", slog.LevelInfo}, + {"warn", slog.LevelWarn}, + {"warning", slog.LevelWarn}, + {"error", slog.LevelError}, + {"", slog.LevelInfo}, + {"bogus", slog.LevelInfo}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + require.Equal(t, tt.want, parseLevel(tt.in)) + }) + } +} + +func TestRedactSecrets(t *testing.T) { + var buf bytes.Buffer + log := NewWithWriter(&buf, "info") + log.Info("test", slog.String("agent_key", "bkpy_live_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), + slog.String("password", "hunter2"), + slog.String("user", "alice")) + + var got map[string]any + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + require.Equal(t, "***", got["agent_key"]) + require.Equal(t, "***", got["password"]) + require.Equal(t, "alice", got["user"]) +} + +func TestNew_EmitsJSON(t *testing.T) { + var buf bytes.Buffer + log := NewWithWriter(&buf, "info") + log.Info("hello", slog.String("k", "v")) + require.True(t, strings.HasPrefix(strings.TrimSpace(buf.String()), "{"), "expected JSON output") +} diff --git a/apps/agent/internal/metrics/metrics.go b/apps/agent/internal/metrics/metrics.go new file mode 100644 index 0000000..9f350b8 --- /dev/null +++ b/apps/agent/internal/metrics/metrics.go @@ -0,0 +1,113 @@ +// Package metrics owns the Prometheus instrumentation surface for the +// agent. +// +// Security: the HTTP endpoint is intended to be bound to LOOPBACK only +// (default 127.0.0.1:9090). Operators who want external scraping +// should configure their Prometheus to scrape via an SSH tunnel or +// expose via a reverse proxy with authentication. NEVER bind to +// 0.0.0.0 in production — the endpoint reveals job IDs, run cadence, +// and other metadata an attacker can use to fingerprint the host. +// +// Metric names follow Prometheus best practice: `backupy_agent_*` +// prefix, base unit (seconds, bytes), and `_total` suffix for +// monotonic counters. +// +// Counters/gauges/histograms are package-level singletons registered +// in init() so call sites can increment them inline without nil +// checks or dependency injection. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // RunsTotal counts backup runs by job_id and terminal status + // ("success" or "failure"). Cardinality scales with the number of + // configured jobs — bounded by the user's plan, so safe. + RunsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "backupy_agent_runs_total", + Help: "Total number of backup runs the agent has executed, partitioned by job_id and terminal status.", + }, []string{"job_id", "status"}) + + // RunDuration observes the wall-clock duration of a backup run in + // seconds, partitioned by job_id. Buckets target the documented + // run size envelope (1s small dump → 1h large). + RunDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "backupy_agent_run_duration_seconds", + Help: "Wall-clock duration of backup runs in seconds.", + Buckets: []float64{1, 5, 15, 60, 300, 900, 3600}, + }, []string{"job_id"}) + + // RunSizeBytes observes the size of the uploaded ciphertext in + // bytes, partitioned by job_id. Buckets span 1 MiB → 10 GiB. + RunSizeBytes = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "backupy_agent_run_size_bytes", + Help: "Size of the uploaded ciphertext in bytes.", + Buckets: []float64{1 << 20, 10 << 20, 100 << 20, 1 << 30, 10 << 30}, + }, []string{"job_id"}) + + // WSSState is 1 for the currently-active state and 0 otherwise. + // Allowed states: connected, reconnecting, disconnected. + WSSState = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backupy_agent_wss_connection_state", + Help: "WSS connection state — 1 for the current state, 0 for the others. Labels: state=connected|reconnecting|disconnected.", + }, []string{"state"}) + + // WSSReconnects counts every reconnect attempt since the agent + // started. + WSSReconnects = promauto.NewCounter(prometheus.CounterOpts{ + Name: "backupy_agent_wss_reconnects_total", + Help: "Total number of WSS reconnect attempts since process start.", + }) + + // DispatchPending tracks the on-disk persistent queue depth. + DispatchPending = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "backupy_agent_dispatch_pending", + Help: "Current depth of the persistent dispatch queue.", + }) + + // BuildInfo is a 1-valued gauge labelled with version + commit so + // dashboards can group by build. + BuildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backupy_agent_build_info", + Help: "Always 1; labels expose agent build version and commit.", + }, []string{"version", "commit"}) +) + +// Reset clears every metric value. Intended for tests only — the +// production singletons accumulate across the process lifetime. +func Reset() { + RunsTotal.Reset() + RunDuration.Reset() + RunSizeBytes.Reset() + WSSState.Reset() + BuildInfo.Reset() + DispatchPending.Set(0) +} + +// SetWSSState marks `state` as the active connection state by writing +// 1 to its gauge and zeroing the other two. The caller passes one of +// "connected", "reconnecting", "disconnected" — unknown values are +// recorded as-is so a misuse is visible in /metrics rather than +// silently swallowed. +func SetWSSState(state string) { + for _, s := range []string{"connected", "reconnecting", "disconnected"} { + v := 0.0 + if s == state { + v = 1.0 + } + WSSState.WithLabelValues(s).Set(v) + } + if state != "connected" && state != "reconnecting" && state != "disconnected" { + WSSState.WithLabelValues(state).Set(1) + } +} + +// SetBuildInfo registers the build version + commit as labels on the +// build_info gauge. Safe to call multiple times — the gauge value is +// always 1. +func SetBuildInfo(version, commit string) { + BuildInfo.WithLabelValues(version, commit).Set(1) +} diff --git a/apps/agent/internal/metrics/metrics_test.go b/apps/agent/internal/metrics/metrics_test.go new file mode 100644 index 0000000..8124819 --- /dev/null +++ b/apps/agent/internal/metrics/metrics_test.go @@ -0,0 +1,106 @@ +package metrics + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/stretchr/testify/require" +) + +func TestHandlerExposesBuildInfo(t *testing.T) { + Reset() + SetBuildInfo("v1.2.3", "abcdef") + + srv := httptest.NewServer(promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + out := string(body) + + require.Contains(t, out, "backupy_agent_build_info") + require.Contains(t, out, `version="v1.2.3"`) + require.Contains(t, out, `commit="abcdef"`) +} + +func TestSetWSSStateOneHot(t *testing.T) { + Reset() + SetWSSState("connected") + + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + var connected, reconnecting, disconnected float64 + for _, mf := range mfs { + if mf.GetName() != "backupy_agent_wss_connection_state" { + continue + } + for _, m := range mf.GetMetric() { + var label string + for _, l := range m.GetLabel() { + if l.GetName() == "state" { + label = l.GetValue() + } + } + switch label { + case "connected": + connected = m.GetGauge().GetValue() + case "reconnecting": + reconnecting = m.GetGauge().GetValue() + case "disconnected": + disconnected = m.GetGauge().GetValue() + } + } + } + require.Equal(t, 1.0, connected) + require.Equal(t, 0.0, reconnecting) + require.Equal(t, 0.0, disconnected) +} + +func TestListenAndServeContextCancel(t *testing.T) { + // Ask the OS for a free port to avoid collisions with a real + // running agent on 9090. + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := ln.Addr().String() + require.NoError(t, ln.Close()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- ListenAndServe(ctx, addr) }() + + // Wait briefly for the listener to come up. + var resp *http.Response + for i := 0; i < 50; i++ { + resp, err = http.Get("http://" + addr + "/metrics") + if err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.True(t, strings.Contains(string(body), "backupy_agent_"), "expected backupy_agent_* metrics in body") + + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("metrics server did not shut down within 5s of ctx cancel") + } +} diff --git a/apps/agent/internal/metrics/server.go b/apps/agent/internal/metrics/server.go new file mode 100644 index 0000000..d8f6aec --- /dev/null +++ b/apps/agent/internal/metrics/server.go @@ -0,0 +1,66 @@ +package metrics + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// DefaultAddr is the loopback-only default address the operator should +// stick with unless they have a specific reason to override it. +const DefaultAddr = "127.0.0.1:9090" + +// ListenAndServe binds `addr` and serves `/metrics` over plain HTTP +// using the default Prometheus gatherer. The HTTP server is shut down +// gracefully when `ctx` is cancelled. +// +// `addr` MUST resolve to a loopback interface in production. The +// function does not enforce this — operators may need to expose +// metrics over a private network behind a reverse proxy — but the +// package documentation calls out the policy and the default in +// DefaultAddr is loopback. +func ListenAndServe(ctx context.Context, addr string) error { + if addr == "" { + addr = DefaultAddr + } + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{})) + + srv := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("metrics: listen on %s: %w", addr, err) + } + + errCh := make(chan error, 1) + go func() { + serveErr := srv.Serve(ln) + if serveErr != nil && !errors.Is(serveErr, http.ErrServerClosed) { + errCh <- serveErr + return + } + errCh <- nil + }() + + select { + case <-ctx.Done(): + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutCtx) + <-errCh + return nil + case err := <-errCh: + return err + } +} diff --git a/apps/agent/internal/pipeline/compress.go b/apps/agent/internal/pipeline/compress.go new file mode 100644 index 0000000..8b51a0b --- /dev/null +++ b/apps/agent/internal/pipeline/compress.go @@ -0,0 +1,65 @@ +package pipeline + +import ( + "fmt" + "io" + + "github.com/klauspost/compress/zstd" +) + +// CompressZstd streams `in` through a zstd encoder into `out`. +// +// Returns: +// - originalBytes : plaintext bytes consumed from `in` +// - compressedBytes: zstd-framed bytes written to `out` +// +// The encoder is created with the default level (SpeedDefault, ~level 3) — +// a sensible balance between ratio and CPU for streaming DB dumps. Callers +// who need a different level should use the lower-level zstdWriter directly. +func CompressZstd(in io.Reader, out io.Writer) (int64, int64, error) { + cw := &countingWriter{w: out} + enc, err := zstd.NewWriter(cw, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return 0, 0, fmt.Errorf("pipeline: zstd new writer: %w", err) + } + // NOTE: do NOT defer enc.Close() here. klauspost's zstd writer emits + // an additional empty frame on a second Close() — if we both defer and + // close explicitly, the trailing 4 bytes corrupt the downstream stream + // and skew `cw.n`. We close exactly once below and ensure the encoder + // is released on error paths too. + + n, err := io.Copy(enc, in) + if err != nil { + _ = enc.Close() + return n, cw.n, fmt.Errorf("pipeline: zstd copy: %w", err) + } + if err := enc.Close(); err != nil { + return n, cw.n, fmt.Errorf("pipeline: zstd close: %w", err) + } + return n, cw.n, nil +} + +// NewZstdWriter wraps `out` in a zstd encoder. Callers MUST Close the +// returned writer before the stream is considered final — the trailer +// would otherwise be missing. +func NewZstdWriter(out io.Writer) (io.WriteCloser, error) { + enc, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return nil, fmt.Errorf("pipeline: zstd new writer: %w", err) + } + return enc, nil +} + +// countingWriter is an io.Writer that counts bytes written to the +// wrapped writer. Used to measure compressed output size without +// double-buffering. +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} diff --git a/apps/agent/internal/pipeline/compress_test.go b/apps/agent/internal/pipeline/compress_test.go new file mode 100644 index 0000000..fb7a566 --- /dev/null +++ b/apps/agent/internal/pipeline/compress_test.go @@ -0,0 +1,57 @@ +package pipeline + +import ( + "bytes" + "crypto/rand" + "io" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" +) + +func TestCompressZstd_RoundTrip(t *testing.T) { + plaintext := make([]byte, 1<<20) + _, err := rand.Read(plaintext) + require.NoError(t, err) + + var compressed bytes.Buffer + orig, ccount, err := CompressZstd(bytes.NewReader(plaintext), &compressed) + require.NoError(t, err) + require.Equal(t, int64(len(plaintext)), orig) + require.Equal(t, int64(compressed.Len()), ccount) + + dec, err := zstd.NewReader(&compressed) + require.NoError(t, err) + defer dec.Close() + round, err := io.ReadAll(dec) + require.NoError(t, err) + require.Equal(t, plaintext, round, "decompressed bytes must match the original") +} + +func TestCompressZstd_Empty(t *testing.T) { + var compressed bytes.Buffer + orig, ccount, err := CompressZstd(bytes.NewReader(nil), &compressed) + require.NoError(t, err) + require.Equal(t, int64(0), orig) + // klauspost/compress >=1.17 emits zero bytes for an empty input + // (no frame header is written when nothing was buffered). We just + // require ccount to match the bytes actually written to `compressed` + // and the round-trip to decompress cleanly to an empty payload. + require.Equal(t, int64(compressed.Len()), ccount) + + dec, err := zstd.NewReader(&compressed) + require.NoError(t, err) + defer dec.Close() + round, err := io.ReadAll(dec) + require.NoError(t, err) + require.Empty(t, round, "empty input must round-trip to empty output") +} + +func TestCompressZstd_CompressibleInputShrinks(t *testing.T) { + plaintext := bytes.Repeat([]byte("ABCDEFGH"), 4096) // 32 KiB of repetition + var compressed bytes.Buffer + _, _, err := CompressZstd(bytes.NewReader(plaintext), &compressed) + require.NoError(t, err) + require.Less(t, compressed.Len(), len(plaintext)/4, "repetitive input must compress >4×") +} diff --git a/apps/agent/internal/pipeline/encrypt.go b/apps/agent/internal/pipeline/encrypt.go new file mode 100644 index 0000000..cef1c2f --- /dev/null +++ b/apps/agent/internal/pipeline/encrypt.go @@ -0,0 +1,181 @@ +// Backupy chunk-stream encryption format (v1). +// +// The agent encrypts the (already-compressed) backup as a sequence of +// independent AES-256-GCM frames. The output is appended to the upload +// stream byte-by-byte — no length prefix on the whole blob, no header. +// +// Wire format (all integers big-endian): +// +// chunk := uint32 ciphertext_len // bytes that follow, EXCLUDING this u32 +// 12-byte random nonce // unique per chunk +// ciphertext (≤ chunkPlainSize + 16-byte GCM tag) +// +// EOF marker: a single chunk with ciphertext_len == 0. The decryptor +// treats this as "stream finished cleanly" — without it, a truncated +// upload would be indistinguishable from a clean end. +// +// Chunk plaintext size is CHUNK_PLAIN_SIZE = 1 MiB. Larger frames waste +// memory on the decryptor; smaller frames pay too much per-chunk overhead. +// +// The DEK is 32 bytes (AES-256). It is supplied by the server in +// RunBackup.encrypted_dek, decrypted by the agent runtime (KMS path — +// not covered here), and discarded once the upload completes. The +// envelope ciphertext (KMS-wrapped DEK) is passed THROUGH the agent +// unchanged in BackupCompleted.encrypted_dek so the server can persist +// it alongside the backup row. +// +// The backupy-decrypt CLI inverts this format. Keep the constants below +// in sync with apps/backupy-decrypt — they are the single source of truth. + +package pipeline + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" +) + +const ( + // ChunkPlainSize is the maximum plaintext bytes per AES-GCM frame. + // 1 MiB keeps decryption memory bounded while keeping per-chunk + // overhead (16 byte tag + 12 byte nonce + 4 byte header) negligible. + ChunkPlainSize = 1 << 20 + + // dekSize is the expected DEK length (AES-256). + dekSize = 32 + + // nonceSize matches AES-GCM's standard 96-bit nonce. The stdlib + // rejects any other length when GCM.NonceSize() is honoured. + nonceSize = 12 + + // gcmTagSize is the GCM authentication tag length (16 bytes). + gcmTagSize = 16 + + // chunkHeaderSize is the 4-byte big-endian length prefix per chunk. + chunkHeaderSize = 4 +) + +// Encryptor encrypts arbitrarily large streams using AES-256-GCM with a +// per-chunk random nonce. Construct one per backup run — reuse across +// runs is allowed but cheap to avoid. +type Encryptor struct { + dek []byte + aead cipher.AEAD +} + +// NewEncryptor builds an Encryptor from a 32-byte DEK. +func NewEncryptor(dek []byte) (*Encryptor, error) { + if len(dek) != dekSize { + return nil, fmt.Errorf("pipeline: DEK must be %d bytes, got %d", dekSize, len(dek)) + } + block, err := aes.NewCipher(dek) + if err != nil { + return nil, fmt.Errorf("pipeline: aes new cipher: %w", err) + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("pipeline: gcm new: %w", err) + } + // Defensive: confirm the stdlib's nonce expectation matches our constant. + if aead.NonceSize() != nonceSize { + return nil, fmt.Errorf("pipeline: gcm nonce size mismatch: %d", aead.NonceSize()) + } + // Keep a copy so the caller may overwrite the slice afterwards. + keyCopy := make([]byte, len(dek)) + copy(keyCopy, dek) + return &Encryptor{dek: keyCopy, aead: aead}, nil +} + +// Stream reads plaintext from `in` in ChunkPlainSize chunks, encrypts +// each one with a fresh random nonce, and writes framed ciphertext to +// `out`. After EOF on `in` it writes the zero-length terminator chunk. +// +// Returns the total number of PLAINTEXT bytes consumed from `in`. +func (e *Encryptor) Stream(in io.Reader, out io.Writer) (int64, error) { + if e == nil || e.aead == nil { + return 0, errors.New("pipeline: nil Encryptor") + } + buf := make([]byte, ChunkPlainSize) + header := make([]byte, chunkHeaderSize) + nonce := make([]byte, nonceSize) + var total int64 + + for { + n, readErr := io.ReadFull(in, buf) + if n > 0 { + if _, err := rand.Read(nonce); err != nil { + return total, fmt.Errorf("pipeline: read nonce: %w", err) + } + ct := e.aead.Seal(nil, nonce, buf[:n], nil) + // Frame: u32(len) || nonce || ciphertext+tag + binary.BigEndian.PutUint32(header, uint32(len(nonce)+len(ct))) + if _, err := out.Write(header); err != nil { + return total, fmt.Errorf("pipeline: write chunk header: %w", err) + } + if _, err := out.Write(nonce); err != nil { + return total, fmt.Errorf("pipeline: write chunk nonce: %w", err) + } + if _, err := out.Write(ct); err != nil { + return total, fmt.Errorf("pipeline: write chunk ciphertext: %w", err) + } + total += int64(n) + } + if readErr == io.EOF || readErr == io.ErrUnexpectedEOF { + break + } + if readErr != nil { + return total, fmt.Errorf("pipeline: read plaintext: %w", readErr) + } + } + + // EOF marker: zero-length chunk. + binary.BigEndian.PutUint32(header, 0) + if _, err := out.Write(header); err != nil { + return total, fmt.Errorf("pipeline: write eof marker: %w", err) + } + return total, nil +} + +// Decrypt is the inverse of Stream — used by tests and the +// backupy-decrypt CLI. Validates GCM tags and the EOF marker. +func (e *Encryptor) Decrypt(in io.Reader, out io.Writer) (int64, error) { + if e == nil || e.aead == nil { + return 0, errors.New("pipeline: nil Encryptor") + } + header := make([]byte, chunkHeaderSize) + var total int64 + for { + if _, err := io.ReadFull(in, header); err != nil { + if errors.Is(err, io.EOF) { + // Stream ended without an explicit terminator — refuse. + return total, errors.New("pipeline: encrypted stream truncated (no EOF marker)") + } + return total, fmt.Errorf("pipeline: read chunk header: %w", err) + } + size := binary.BigEndian.Uint32(header) + if size == 0 { + return total, nil // clean EOF + } + if size < uint32(nonceSize+gcmTagSize) { + return total, fmt.Errorf("pipeline: chunk size %d below minimum", size) + } + frame := make([]byte, size) + if _, err := io.ReadFull(in, frame); err != nil { + return total, fmt.Errorf("pipeline: read chunk body: %w", err) + } + nonce := frame[:nonceSize] + ct := frame[nonceSize:] + pt, err := e.aead.Open(nil, nonce, ct, nil) + if err != nil { + return total, fmt.Errorf("pipeline: gcm open: %w", err) + } + if _, err := out.Write(pt); err != nil { + return total, fmt.Errorf("pipeline: write plaintext: %w", err) + } + total += int64(len(pt)) + } +} diff --git a/apps/agent/internal/pipeline/encrypt_test.go b/apps/agent/internal/pipeline/encrypt_test.go new file mode 100644 index 0000000..bfe1938 --- /dev/null +++ b/apps/agent/internal/pipeline/encrypt_test.go @@ -0,0 +1,116 @@ +package pipeline + +import ( + "bytes" + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func newDEK(t *testing.T) []byte { + t.Helper() + dek := make([]byte, 32) + _, err := rand.Read(dek) + require.NoError(t, err) + return dek +} + +func TestEncryptor_RoundTrip_5MB(t *testing.T) { + plaintext := make([]byte, 5<<20) + _, err := rand.Read(plaintext) + require.NoError(t, err) + + e, err := NewEncryptor(newDEK(t)) + require.NoError(t, err) + + var ct bytes.Buffer + consumed, err := e.Stream(bytes.NewReader(plaintext), &ct) + require.NoError(t, err) + require.Equal(t, int64(len(plaintext)), consumed) + + var round bytes.Buffer + pt, err := e.Decrypt(&ct, &round) + require.NoError(t, err) + require.Equal(t, int64(len(plaintext)), pt) + require.Equal(t, plaintext, round.Bytes()) +} + +func TestEncryptor_RoundTrip_Empty(t *testing.T) { + e, err := NewEncryptor(newDEK(t)) + require.NoError(t, err) + + var ct bytes.Buffer + consumed, err := e.Stream(bytes.NewReader(nil), &ct) + require.NoError(t, err) + require.Equal(t, int64(0), consumed) + + var round bytes.Buffer + pt, err := e.Decrypt(&ct, &round) + require.NoError(t, err) + require.Equal(t, int64(0), pt) + require.Empty(t, round.Bytes()) +} + +func TestEncryptor_WrongKeyFails(t *testing.T) { + plaintext := bytes.Repeat([]byte("hello"), 1000) + e1, err := NewEncryptor(newDEK(t)) + require.NoError(t, err) + e2, err := NewEncryptor(newDEK(t)) + require.NoError(t, err) + + var ct bytes.Buffer + _, err = e1.Stream(bytes.NewReader(plaintext), &ct) + require.NoError(t, err) + + var round bytes.Buffer + _, err = e2.Decrypt(&ct, &round) + require.Error(t, err, "decrypt with a different DEK must fail GCM tag check") +} + +func TestEncryptor_TamperingDetected(t *testing.T) { + dek := newDEK(t) + e, err := NewEncryptor(dek) + require.NoError(t, err) + + plaintext := bytes.Repeat([]byte("abcd"), 4096) + var ct bytes.Buffer + _, err = e.Stream(bytes.NewReader(plaintext), &ct) + require.NoError(t, err) + + // Flip a bit inside the first ciphertext chunk (skip header + nonce). + tampered := ct.Bytes() + flipAt := chunkHeaderSize + nonceSize + 1 + require.Greater(t, len(tampered), flipAt) + tampered[flipAt] ^= 0x01 + + var round bytes.Buffer + _, err = e.Decrypt(bytes.NewReader(tampered), &round) + require.Error(t, err, "bit-flip in ciphertext must trigger a GCM tag failure") +} + +func TestEncryptor_RejectsBadDEKLength(t *testing.T) { + _, err := NewEncryptor(make([]byte, 16)) + require.Error(t, err) + _, err = NewEncryptor(make([]byte, 31)) + require.Error(t, err) + _, err = NewEncryptor(make([]byte, 33)) + require.Error(t, err) +} + +func TestEncryptor_TruncatedStreamFails(t *testing.T) { + e, err := NewEncryptor(newDEK(t)) + require.NoError(t, err) + + var ct bytes.Buffer + _, err = e.Stream(bytes.NewReader(bytes.Repeat([]byte("X"), 4096)), &ct) + require.NoError(t, err) + + // Drop the last 4 bytes of the EOF marker. + b := ct.Bytes() + truncated := b[:len(b)-2] + + var round bytes.Buffer + _, err = e.Decrypt(bytes.NewReader(truncated), &round) + require.Error(t, err, "missing EOF marker must be detected") +} diff --git a/apps/agent/internal/pipeline/hooks.go b/apps/agent/internal/pipeline/hooks.go new file mode 100644 index 0000000..9453693 --- /dev/null +++ b/apps/agent/internal/pipeline/hooks.go @@ -0,0 +1,283 @@ +// Package pipeline — pre/post hook execution (D-16). +// +// # Security model +// +// Pre and post hooks are arbitrary shell commands executed by the agent +// on its own host with the agent's filesystem permissions. They are +// inherently dangerous: any code path that can write a hook string into +// backup_jobs can run code on every agent that polls that job. +// +// The agent host owner must trust the user's backup config — a +// compromised server could push hostile hooks. We mitigate this with +// defense-in-depth limits enforced on BOTH sides: +// +// server: validates max command length (HookCommandMaxBytes) +// and hook count (HooksMaxCount) at job-config time. +// agent : enforces a per-hook timeout (DefaultHookTimeout, capped by +// HookTimeoutMax) and a hard total budget per backup run +// (HooksTotalBudget) so a wedged hook cannot keep an agent +// process pinned forever. +// +// Commands are passed verbatim to /bin/sh -c — NO env-var or path +// expansion happens in our code. The shell performs interpolation; we +// never call fmt.Sprintf-style formatting on user-supplied strings. +// +// Hook stdout and stderr are captured into separate 8 KB ring buffers +// (HookOutputBufBytes) so a noisy hook cannot OOM the agent. +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os/exec" + "sync" + "syscall" + "time" +) + +// Limits — these constants encode the security model above and are +// referenced by the server-side validators. +const ( + // HookCommandMaxBytes caps each hook string. 4 KB matches the + // argv/env headroom on every supported OS and is well above any + // realistic shell-pipeline length. + HookCommandMaxBytes = 4 * 1024 + + // HooksMaxCount caps the number of pre/post hooks per job. 16 is + // generous for any normal workflow (snapshot → quiesce → notify), + // while preventing a runaway config from generating dozens of + // child processes per run. + HooksMaxCount = 16 + + // HookOutputBufBytes is the per-stream (stdout, stderr) ring-buffer + // size. 8 KB is small enough to keep many concurrent runs in + // memory and large enough to capture typical hook chatter. + HookOutputBufBytes = 8 * 1024 + + // DefaultHookTimeout is the per-hook timeout when the job config + // does not override it. + DefaultHookTimeout = 5 * time.Minute + + // HookTimeoutMax caps the per-hook timeout regardless of job + // config. Prevents a single hostile hook from hanging the agent. + HookTimeoutMax = 15 * time.Minute + + // HooksTotalBudget is the hard ceiling for the combined runtime of + // every pre+post hook in a single backup run. Once exceeded, + // further hooks return immediately with ErrHooksBudgetExceeded. + HooksTotalBudget = 30 * time.Minute +) + +// ErrHooksBudgetExceeded indicates the total hook runtime budget for +// a backup run was exhausted. +var ErrHooksBudgetExceeded = errors.New("pipeline: hook budget exceeded for run") + +// HookResult is the post-mortem of a single hook invocation. Stdout and +// Stderr are best-effort: the last HookOutputBufBytes of each stream are +// kept, earlier bytes are dropped. +type HookResult struct { + // Command is the raw shell string that was executed (informational). + Command string + // ExitCode is the process exit code. 0 == success. For timeouts + // and context cancellations it is -1. + ExitCode int + // Stdout / Stderr hold up to HookOutputBufBytes of captured output. + Stdout string + Stderr string + // Duration is the wall-clock time the hook took. + Duration time.Duration + // TimedOut indicates the hook was killed because its per-hook + // timeout fired (vs. caller-cancelled or completed naturally). + TimedOut bool +} + +// RunHook executes command under /bin/sh -c, applying timeout and +// capturing up to HookOutputBufBytes of stdout/stderr each. +// +// The returned error is non-nil when the hook FAILED (non-zero exit, +// timeout, or process spawn error). HookResult is always returned with +// whatever fields are known. +// +// Environment variables in env are added on top of the current process +// environment (caller can pass nil for default). +func RunHook(ctx context.Context, command string, env []string, timeout time.Duration) (HookResult, error) { + if command == "" { + return HookResult{}, errors.New("pipeline: empty hook command") + } + if timeout <= 0 { + timeout = DefaultHookTimeout + } + if timeout > HookTimeoutMax { + timeout = HookTimeoutMax + } + + // Per-hook timeout layered on top of the caller's ctx. + hookCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(hookCtx, "/bin/sh", "-c", command) + if env != nil { + // Append (not replace) so the agent's PATH etc. are still + // available to the shell. + cmd.Env = append(cmd.Env, env...) + } + + stdoutBuf := newHookRingBuffer(HookOutputBufBytes) + stderrBuf := newHookRingBuffer(HookOutputBufBytes) + cmd.Stdout = stdoutBuf + cmd.Stderr = stderrBuf + + start := time.Now() + runErr := cmd.Run() + dur := time.Since(start) + + result := HookResult{ + Command: command, + Duration: dur, + Stdout: stdoutBuf.String(), + Stderr: stderrBuf.String(), + } + + if runErr == nil { + result.ExitCode = 0 + return result, nil + } + + // Distinguish: timeout / parent-cancel / non-zero exit. + if hookCtx.Err() != nil { + // Either deadline (timeout) or caller cancel. Mark exit -1. + result.ExitCode = -1 + if errors.Is(hookCtx.Err(), context.DeadlineExceeded) { + result.TimedOut = true + return result, fmt.Errorf("hook timed out after %s: %w", timeout, hookCtx.Err()) + } + return result, fmt.Errorf("hook cancelled: %w", hookCtx.Err()) + } + + // Non-zero exit (or process-start failure). exec.ExitError carries + // the exit code. + var exitErr *exec.ExitError + if errors.As(runErr, &exitErr) { + if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok { + result.ExitCode = ws.ExitStatus() + } else { + result.ExitCode = exitErr.ExitCode() + } + return result, fmt.Errorf("hook exited non-zero (%d): %w", result.ExitCode, runErr) + } + // Spawn failure, signal, or unknown error. + result.ExitCode = -1 + return result, fmt.Errorf("hook failed: %w", runErr) +} + +// HookSet executes a sequence of hooks under a shared total-budget. +// It returns the per-hook results in order, and the first error (if +// any). The shared budget across the whole set is HooksTotalBudget; +// callers should hold one HookSet per backup run and feed it the +// pre_hooks list, then the post_hooks list. +type HookSet struct { + mu sync.Mutex + consumed time.Duration +} + +// NewHookSet returns an empty HookSet that has not yet consumed any +// budget. +func NewHookSet() *HookSet { + return &HookSet{} +} + +// Run executes one hook, charging its duration against the set's +// budget. If the budget is already exhausted when Run is called, the +// hook is skipped and ErrHooksBudgetExceeded is returned with an empty +// HookResult. +func (h *HookSet) Run(ctx context.Context, command string, env []string, timeout time.Duration) (HookResult, error) { + h.mu.Lock() + used := h.consumed + h.mu.Unlock() + if used >= HooksTotalBudget { + return HookResult{Command: command}, ErrHooksBudgetExceeded + } + // Cap the per-hook timeout at the remaining budget. + remaining := HooksTotalBudget - used + if timeout <= 0 || timeout > remaining { + timeout = remaining + } + res, err := RunHook(ctx, command, env, timeout) + h.mu.Lock() + h.consumed += res.Duration + h.mu.Unlock() + return res, err +} + +// hookRingBuffer keeps the LAST `cap` bytes written to it. Writes that +// exceed `cap` discard the oldest bytes. Safe for the io.Writer +// contract used by exec.Cmd; not safe for concurrent writes (exec.Cmd +// writes from one goroutine per stream). +type hookRingBuffer struct { + buf []byte + cap int + full bool + pos int // next write position +} + +func newHookRingBuffer(cap int) *hookRingBuffer { + return &hookRingBuffer{buf: make([]byte, 0, cap), cap: cap} +} + +func (r *hookRingBuffer) Write(p []byte) (int, error) { + n := len(p) + if n == 0 { + return 0, nil + } + // If we haven't yet wrapped, grow the slice up to cap. + if !r.full && len(r.buf) < r.cap { + room := r.cap - len(r.buf) + if n <= room { + r.buf = append(r.buf, p...) + r.pos = (r.pos + n) % r.cap + if len(r.buf) == r.cap { + r.full = true + } + return n, nil + } + r.buf = append(r.buf, p[:room]...) + p = p[room:] + r.full = true + r.pos = 0 + } + // We're full; overwrite oldest bytes. + if len(p) >= r.cap { + // Only the trailing cap bytes matter. + copy(r.buf, p[len(p)-r.cap:]) + r.pos = 0 + return n, nil + } + // Write may wrap around the end of the slice. + end := r.pos + len(p) + if end <= r.cap { + copy(r.buf[r.pos:], p) + } else { + first := r.cap - r.pos + copy(r.buf[r.pos:], p[:first]) + copy(r.buf[:len(p)-first], p[first:]) + } + r.pos = (r.pos + len(p)) % r.cap + return n, nil +} + +// String returns the buffer contents in write order (oldest first). +func (r *hookRingBuffer) String() string { + if !r.full { + return string(r.buf) + } + out := bytes.NewBuffer(make([]byte, 0, r.cap)) + out.Write(r.buf[r.pos:]) + out.Write(r.buf[:r.pos]) + return out.String() +} + +// Ensure hookRingBuffer satisfies io.Writer. +var _ io.Writer = (*hookRingBuffer)(nil) diff --git a/apps/agent/internal/pipeline/hooks_test.go b/apps/agent/internal/pipeline/hooks_test.go new file mode 100644 index 0000000..9388607 --- /dev/null +++ b/apps/agent/internal/pipeline/hooks_test.go @@ -0,0 +1,202 @@ +package pipeline + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestRunHook_Success(t *testing.T) { + res, err := RunHook(context.Background(), "echo hi", nil, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.ExitCode != 0 { + t.Errorf("ExitCode=%d want 0", res.ExitCode) + } + if res.Stdout != "hi\n" { + t.Errorf("Stdout=%q want %q", res.Stdout, "hi\n") + } + if res.TimedOut { + t.Errorf("TimedOut=true; should be false on success") + } + if res.Duration <= 0 { + t.Errorf("Duration=%v should be > 0", res.Duration) + } +} + +func TestRunHook_NonZeroExit(t *testing.T) { + res, err := RunHook(context.Background(), "false", nil, time.Second) + if err == nil { + t.Fatalf("expected error for non-zero exit") + } + if res.ExitCode != 1 { + t.Errorf("ExitCode=%d want 1", res.ExitCode) + } + if res.TimedOut { + t.Errorf("TimedOut should be false for plain non-zero exit") + } +} + +func TestRunHook_Timeout(t *testing.T) { + start := time.Now() + res, err := RunHook(context.Background(), "sleep 10", nil, 100*time.Millisecond) + elapsed := time.Since(start) + if err == nil { + t.Fatalf("expected timeout error") + } + if !res.TimedOut { + t.Errorf("TimedOut=false; want true") + } + if res.ExitCode != -1 { + t.Errorf("ExitCode=%d want -1 on timeout", res.ExitCode) + } + // Killed within reasonable slack of the timeout. + if elapsed > 5*time.Second { + t.Errorf("hook took %v, should die well under 5s", elapsed) + } +} + +func TestRunHook_ContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var res HookResult + var err error + start := time.Now() + go func() { + res, err = RunHook(ctx, "sleep 30", nil, 30*time.Second) + close(done) + }() + // Let the process actually start before we cancel. + time.Sleep(50 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatalf("hook did not exit within 2s of cancel") + } + if err == nil { + t.Errorf("expected error on cancel") + } + if res.TimedOut { + t.Errorf("cancel should NOT be reported as TimedOut") + } + if res.ExitCode != -1 { + t.Errorf("ExitCode=%d want -1 on cancel", res.ExitCode) + } + // Sanity: we cancelled fast. + if time.Since(start) > 5*time.Second { + t.Errorf("cancel took too long: %v", time.Since(start)) + } +} + +func TestRunHook_StdoutTruncation(t *testing.T) { + // Produce ~100 KB to stdout; ring buffer keeps last 8 KB. + // `yes "X" | head -c 102400` is portable across BSD + GNU userland. + cmd := `yes X | head -c 102400` + res, err := RunHook(context.Background(), cmd, nil, 10*time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.Stdout) != HookOutputBufBytes { + t.Errorf("stdout length=%d want %d (ring buffer cap)", len(res.Stdout), HookOutputBufBytes) + } + // Every byte in the buffer should be a captured 'X' or '\n' (from yes). + for i, c := range res.Stdout { + if c != 'X' && c != '\n' { + t.Errorf("unexpected byte at offset %d: %q", i, c) + break + } + } +} + +func TestRunHook_EnvPassedThrough(t *testing.T) { + res, err := RunHook(context.Background(), `echo "$BACKUPY_TEST_VAR"`, []string{"BACKUPY_TEST_VAR=hello"}, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if strings.TrimSpace(res.Stdout) != "hello" { + t.Errorf("stdout=%q want %q", res.Stdout, "hello") + } +} + +func TestRunHook_EmptyCommandRejected(t *testing.T) { + if _, err := RunHook(context.Background(), "", nil, time.Second); err == nil { + t.Error("empty command must be rejected") + } +} + +func TestRunHook_StderrCaptured(t *testing.T) { + res, err := RunHook(context.Background(), `echo "oops" 1>&2; exit 1`, nil, time.Second) + if err == nil { + t.Fatalf("expected non-zero exit error") + } + if res.ExitCode != 1 { + t.Errorf("ExitCode=%d want 1", res.ExitCode) + } + if strings.TrimSpace(res.Stderr) != "oops" { + t.Errorf("stderr=%q want %q", res.Stderr, "oops") + } +} + +func TestHookSet_BudgetExhaustion(t *testing.T) { + hs := NewHookSet() + // Pre-charge past the budget so the next Run is immediately denied. + hs.consumed = HooksTotalBudget + time.Second + _, err := hs.Run(context.Background(), "true", nil, time.Second) + if err == nil { + t.Error("expected ErrHooksBudgetExceeded after budget consumed") + } +} + +func TestHookSet_NormalRunCharges(t *testing.T) { + hs := NewHookSet() + res, err := hs.Run(context.Background(), "true", nil, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.ExitCode != 0 { + t.Errorf("ExitCode=%d want 0", res.ExitCode) + } + if hs.consumed <= 0 { + t.Errorf("consumed=%v should be > 0 after a run", hs.consumed) + } +} + +func TestHookRingBuffer_Truncation(t *testing.T) { + buf := newHookRingBuffer(8) + // Write more than capacity in chunks. + for _, chunk := range []string{"abcd", "efgh", "ijkl"} { + if _, err := buf.Write([]byte(chunk)); err != nil { + t.Fatalf("Write: %v", err) + } + } + got := buf.String() + if got != "efghijkl" { + t.Errorf("ring buffer kept %q, want %q", got, "efghijkl") + } +} + +func TestHookRingBuffer_SingleHugeWrite(t *testing.T) { + buf := newHookRingBuffer(8) + // Single write much larger than capacity. + src := []byte("0123456789ABCDEF") + if _, err := buf.Write(src); err != nil { + t.Fatalf("Write: %v", err) + } + got := buf.String() + if got != "89ABCDEF" { + t.Errorf("ring buffer kept %q, want %q", got, "89ABCDEF") + } +} + +func TestHookRingBuffer_SmallWriteUnderCap(t *testing.T) { + buf := newHookRingBuffer(16) + if _, err := buf.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + if got := buf.String(); got != "hello" { + t.Errorf("got %q want %q", got, "hello") + } +} diff --git a/apps/agent/internal/pipeline/mongodump.go b/apps/agent/internal/pipeline/mongodump.go new file mode 100644 index 0000000..bbd62ab --- /dev/null +++ b/apps/agent/internal/pipeline/mongodump.go @@ -0,0 +1,219 @@ +// B14: MongoDB driver. +// +// Spawns `mongodump --archive --gzip` and streams the resulting archive +// to the pipeline writer. The archive is a single binary blob containing +// every database (or one when target.Connection.Database is set), so +// downstream stages (zstd compress, AES-GCM encrypt, upload) can treat +// it the same way they treat pg_dump's custom-format archive. +// +// We deliberately shell out instead of importing the official Mongo Go +// driver: the binary handles oplog tailing, BSON encoding and resume +// semantics already, and keeping the agent's go.mod minimal matters. +package pipeline + +import ( + "context" + "errors" + "fmt" + "io" + "os/exec" + "strconv" + "strings" + "sync" + "syscall" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// mongoDump is the MongoDB driver implementation. +type mongoDump struct { + binary string + runner cmdRunner +} + +// NewMongoDump constructs the default driver wired to the bundled +// mongodump binary on $PATH. +func NewMongoDump() Driver { + return &mongoDump{binary: "mongodump", runner: realRunner{}} +} + +// Name implements Driver.Name. +func (m *mongoDump) Name() string { return "mongodump" } + +// Validate verifies the mongodump binary is installed and that the +// configured connection answers a quick ping. We use mongodump's own +// `--version` + a no-op archive write to /dev/null with `--dbpath` skipped +// — instead the cheapest reachability test is just `--version`. A full +// connection probe is left to the Dump call to avoid double round-trips +// on small databases. +func (m *mongoDump) Validate(ctx context.Context, target *backupv1.Target) error { + if target == nil || target.Connection == nil { + return errors.New("pipeline: mongodump: nil target/connection") + } + out, err := m.runner.Output(ctx, m.binary, []string{"--version"}, nil) + if err != nil { + return fmt.Errorf("pipeline: mongodump version probe failed (is mongodump installed?): %w", err) + } + if !strings.Contains(strings.ToLower(string(out)), "mongodump") { + return fmt.Errorf("pipeline: unexpected mongodump --version output: %q", string(out)) + } + return nil +} + +// Dump streams a `mongodump --archive --gzip` binary archive to out. +// We do NOT layer mongodump's own gzip on top of zstd — passing --gzip +// here keeps the archive self-describing and lets restore reach for the +// canonical `mongorestore --gzip --archive=…` workflow. The outer zstd +// stage will still compress the gzip stream (gain is marginal but the +// uniform pipeline shape outweighs the cost). +func (m *mongoDump) Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) { + if target == nil || target.Connection == nil { + return DumpInfo{}, errors.New("pipeline: mongodump: nil target/connection") + } + args := append(m.connArgs(target), "--archive", "--gzip") + if err := m.runner.RunStream(ctx, m.binary, args, nil, out); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: mongodump exec: %w", err) + } + versionOut, vErr := m.runner.Output(ctx, m.binary, []string{"--version"}, nil) + engineVersion := "MongoDB" + if vErr == nil { + engineVersion = parseMongodumpVersion(string(versionOut)) + } + return DumpInfo{EngineVersion: engineVersion}, nil +} + +// connArgs builds the host/port/user/uri flag tuple. If a URI is +// embedded in the host field (mongodb://…) we pass it via --uri, +// otherwise we use --host/--port/--username and rely on +// $MONGODB_PASSWORD via env (set by the runner via password_secret_ref). +func (m *mongoDump) connArgs(t *backupv1.Target) []string { + c := t.Connection + args := []string{} + + if strings.HasPrefix(c.Host, "mongodb://") || strings.HasPrefix(c.Host, "mongodb+srv://") { + args = append(args, "--uri", c.Host) + } else { + if c.Host != "" { + args = append(args, "--host", c.Host) + } + if c.Port != 0 { + args = append(args, "--port", strconv.FormatUint(uint64(c.Port), 10)) + } + if c.Username != "" { + args = append(args, "--username", c.Username) + } + if c.PasswordSecretRef != "" { + // mongodump accepts --password inline. We pass it as an + // argument because mongodump does not honour environment + // variables for the password. + args = append(args, "--password", c.PasswordSecretRef) + args = append(args, "--authenticationDatabase", "admin") + } + } + + if c.Database != "" { + args = append(args, "--db", c.Database) + } + return args +} + +// parseMongodumpVersion extracts the version line from mongodump +// --version output: +// +// "mongodump version: 100.9.5" +// -> "MongoDB Tools 100.9.5" +func parseMongodumpVersion(s string) string { + s = strings.TrimSpace(s) + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(strings.ToLower(line), "mongodump version:") { + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + return "MongoDB Tools " + strings.TrimSpace(parts[1]) + } + } + } + return "MongoDB" +} + +// IsMongodumpArchiveMagic reports whether head starts with mongodump's +// archive magic number. The mongo archive format begins with the +// little-endian uint32 0x8199e26d. With --gzip the outer stream is gzip +// (magic 0x1f 0x8b) so we accept either. +func IsMongodumpArchiveMagic(head []byte) bool { + if len(head) >= 2 && head[0] == 0x1f && head[1] == 0x8b { + return true // gzip + } + if len(head) >= 4 && head[0] == 0x6d && head[1] == 0xe2 && head[2] == 0x99 && head[3] == 0x81 { + return true // raw mongo archive (little-endian 0x8199e26d) + } + return false +} + +// ----------------------------------------------------------------------------- +// streaming runner with stderr ring buffer + ctx-cancel friendliness. +// Used by the mongo, redis and sqlite drivers — kept here so we do not +// disturb the existing realRunner that pg_dump / mysqldump rely on. +// ----------------------------------------------------------------------------- + +// ringBuffer is a fixed-capacity tail buffer for stderr. It always +// retains the LAST `cap` bytes written to it, dropping earlier writes. +// Concurrent Write calls are guarded by mu. +type ringBuffer struct { + mu sync.Mutex + buf []byte + cap int +} + +func newRingBuffer(cap int) *ringBuffer { return &ringBuffer{cap: cap} } + +func (r *ringBuffer) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.buf = append(r.buf, p...) + if len(r.buf) > r.cap { + r.buf = r.buf[len(r.buf)-r.cap:] + } + return len(p), nil +} + +func (r *ringBuffer) Tail() string { + r.mu.Lock() + defer r.mu.Unlock() + return strings.TrimSpace(string(r.buf)) +} + +// streamWithCancel runs a child process, copying its stdout to out and +// capturing the last 4 KB of stderr. On ctx.Done() it sends SIGINT to +// the child (rather than the default SIGKILL) so mongodump / redis-cli +// / sqlite3 get a chance to flush partial output and exit cleanly. +func streamWithCancel(ctx context.Context, name string, args []string, env []string, out io.Writer) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = mergeEnv(env) + cmd.Stdout = out + stderrRing := newRingBuffer(4 * 1024) + cmd.Stderr = stderrRing + cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGINT) } + + if err := cmd.Run(); err != nil { + var ee *exec.ExitError + tail := stderrRing.Tail() + if errors.As(err, &ee) { + return fmt.Errorf("%s exited %d: %w (stderr: %s)", name, ee.ExitCode(), err, tail) + } + return fmt.Errorf("%s: %w (stderr: %s)", name, err, tail) + } + return nil +} + +// streamingRunner is a cmdRunner that uses streamWithCancel for +// RunStream. Its Output method behaves like realRunner.Output. +type streamingRunner struct{} + +func (streamingRunner) Output(ctx context.Context, name string, args []string, env []string) ([]byte, error) { + return realRunner{}.Output(ctx, name, args, env) +} + +func (streamingRunner) RunStream(ctx context.Context, name string, args []string, env []string, out io.Writer) error { + return streamWithCancel(ctx, name, args, env, out) +} diff --git a/apps/agent/internal/pipeline/mongodump_test.go b/apps/agent/internal/pipeline/mongodump_test.go new file mode 100644 index 0000000..d02ed0a --- /dev/null +++ b/apps/agent/internal/pipeline/mongodump_test.go @@ -0,0 +1,149 @@ +package pipeline + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +func TestParseMongodumpVersion(t *testing.T) { + t.Parallel() + got := parseMongodumpVersion("mongodump version: 100.9.5\ngit version: abc\n") + require.Equal(t, "MongoDB Tools 100.9.5", got) + + require.Equal(t, "MongoDB", parseMongodumpVersion("totally unexpected")) +} + +func TestIsMongodumpArchiveMagic(t *testing.T) { + t.Parallel() + require.True(t, IsMongodumpArchiveMagic([]byte{0x1f, 0x8b, 0x08})) // gzip + require.True(t, IsMongodumpArchiveMagic([]byte{0x6d, 0xe2, 0x99, 0x81})) + require.False(t, IsMongodumpArchiveMagic([]byte{0x00, 0x00})) + require.False(t, IsMongodumpArchiveMagic(nil)) +} + +func TestMongoDump_Validate_MissingTarget(t *testing.T) { + t.Parallel() + d := &mongoDump{binary: "mongodump", runner: &mockRunner{}} + require.Error(t, d.Validate(context.Background(), nil)) + require.Error(t, d.Validate(context.Background(), &backupv1.Target{})) +} + +func TestMongoDump_Validate_BinaryMissing(t *testing.T) { + t.Parallel() + mock := &mockRunner{outputResp: map[string][]byte{"--version": []byte("")}} + mock.streamErr = errors.New("exec: \"mongodump\": not found in $PATH") + d := &mongoDump{binary: "mongodump", runner: &errOutputRunner{err: errors.New("not found")}} + target := &backupv1.Target{Type: backupv1.DbType_MONGODB, Connection: &backupv1.ConnectionConfig{Host: "h"}} + err := d.Validate(context.Background(), target) + require.Error(t, err) + require.Contains(t, err.Error(), "version probe failed") +} + +func TestMongoDump_Validate_OK(t *testing.T) { + t.Parallel() + mock := &mockRunner{outputResp: map[string][]byte{"--version": []byte("mongodump version: 100.9.5\n")}} + d := &mongoDump{binary: "mongodump", runner: mock} + target := &backupv1.Target{Type: backupv1.DbType_MONGODB, Connection: &backupv1.ConnectionConfig{Host: "h"}} + require.NoError(t, d.Validate(context.Background(), target)) +} + +func TestMongoDump_Dump_StreamsBytes(t *testing.T) { + t.Parallel() + payload := []byte{0x1f, 0x8b, 0x08, 0x00, 0xde, 0xad, 0xbe, 0xef} + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("mongodump version: 100.9.5")}, + streamResp: payload, + } + d := &mongoDump{binary: "mongodump", runner: mock} + target := &backupv1.Target{ + Type: backupv1.DbType_MONGODB, + Connection: &backupv1.ConnectionConfig{ + Host: "h", Port: 27017, Username: "u", PasswordSecretRef: "p", Database: "mydb", + }, + } + var buf testWriter + info, err := d.Dump(context.Background(), target, &buf) + require.NoError(t, err) + require.Equal(t, "MongoDB Tools 100.9.5", info.EngineVersion) + require.Equal(t, payload, buf.Bytes()) + + // Confirm the right CLI flags were passed. + require.NotEmpty(t, mock.calls) + streamCall := mock.calls[0] + require.Contains(t, streamCall.Args, "--archive") + require.Contains(t, streamCall.Args, "--gzip") + require.Contains(t, streamCall.Args, "--host") + require.Contains(t, streamCall.Args, "--port") + require.Contains(t, streamCall.Args, "--username") + require.Contains(t, streamCall.Args, "--password") + require.Contains(t, streamCall.Args, "--db") + require.Contains(t, streamCall.Args, "mydb") + require.Contains(t, streamCall.Args, "--authenticationDatabase") +} + +func TestMongoDump_Dump_URI(t *testing.T) { + t.Parallel() + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("mongodump version: 100.9.5")}, + streamResp: []byte{0x1f, 0x8b}, + } + d := &mongoDump{binary: "mongodump", runner: mock} + target := &backupv1.Target{ + Type: backupv1.DbType_MONGODB, + Connection: &backupv1.ConnectionConfig{ + Host: "mongodb://user:pw@h:27017/?authSource=admin", + }, + } + var buf testWriter + _, err := d.Dump(context.Background(), target, &buf) + require.NoError(t, err) + require.Contains(t, mock.calls[0].Args, "--uri") + require.Contains(t, mock.calls[0].Args, "mongodb://user:pw@h:27017/?authSource=admin") + require.NotContains(t, mock.calls[0].Args, "--host") +} + +func TestMongoDump_Dump_StreamErrorWraps(t *testing.T) { + t.Parallel() + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("mongodump version: 100.9.5")}, + streamErr: errors.New("boom (stderr: connection refused)"), + } + d := &mongoDump{binary: "mongodump", runner: mock} + target := &backupv1.Target{Type: backupv1.DbType_MONGODB, Connection: &backupv1.ConnectionConfig{Host: "h"}} + var buf testWriter + _, err := d.Dump(context.Background(), target, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), "mongodump exec") +} + +func TestMongoDump_Name(t *testing.T) { + t.Parallel() + require.Equal(t, "mongodump", (&mongoDump{}).Name()) +} + +func TestRingBuffer_RetainsTail(t *testing.T) { + t.Parallel() + r := newRingBuffer(8) + _, _ = r.Write([]byte("0123456789")) + require.Equal(t, "23456789", r.Tail()) + _, _ = r.Write([]byte("abc")) + require.Equal(t, "56789abc", r.Tail()) +} + +// errOutputRunner is a cmdRunner whose Output always errors. Used to +// simulate "binary not found" without touching the filesystem. +type errOutputRunner struct{ err error } + +func (e *errOutputRunner) Output(_ context.Context, _ string, _ []string, _ []string) ([]byte, error) { + return nil, e.err +} + +func (e *errOutputRunner) RunStream(_ context.Context, _ string, _ []string, _ []string, _ io.Writer) error { + return e.err +} diff --git a/apps/agent/internal/pipeline/mysqldump.go b/apps/agent/internal/pipeline/mysqldump.go new file mode 100644 index 0000000..4bacd41 --- /dev/null +++ b/apps/agent/internal/pipeline/mysqldump.go @@ -0,0 +1,152 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// mysqldump implements Driver against the bundled mysqldump binary for +// both MySQL and MariaDB targets. +type mysqldump struct { + binary string + runner cmdRunner +} + +// NewMysqldump constructs the default driver wired to the bundled +// mysqldump binary on $PATH. +func NewMysqldump() Driver { + return &mysqldump{binary: "mysqldump", runner: realRunner{}} +} + +// Name implements Driver.Name. +func (m *mysqldump) Name() string { return "mysqldump" } + +// Validate runs --version and a no-data smoke dump (`--no-data --no-create-info`) +// which contacts the server but emits almost nothing. +func (m *mysqldump) Validate(ctx context.Context, target *backupv1.Target) error { + if target == nil || target.Connection == nil { + return errors.New("pipeline: mysqldump: nil target/connection") + } + versionOut, err := m.runner.Output(ctx, m.binary, []string{"--version"}, nil) + if err != nil { + return fmt.Errorf("pipeline: mysqldump version probe failed: %w", err) + } + if !strings.Contains(strings.ToLower(string(versionOut)), "mysqldump") { + return fmt.Errorf("pipeline: unexpected mysqldump --version output: %q", string(versionOut)) + } + args := append(m.connArgs(target), "--no-data", "--no-create-info", "--skip-triggers", "--skip-comments") + if _, err := m.runner.Output(ctx, m.binary, args, nil); err != nil { + return fmt.Errorf("pipeline: mysqldump smoke probe failed: %w", err) + } + return nil +} + +// Dump streams a logical mysqldump SQL stream to `out`. +func (m *mysqldump) Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) { + if target == nil || target.Connection == nil { + return DumpInfo{}, errors.New("pipeline: mysqldump: nil target/connection") + } + args := append(m.connArgs(target), + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--quick", + "--hex-blob", + "--skip-extended-insert", + ) + if err := m.runner.RunStream(ctx, m.binary, args, nil, out); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: mysqldump exec: %w", err) + } + versionOut, vErr := m.runner.Output(ctx, m.binary, []string{"--version"}, nil) + engineVersion := "MySQL" + if vErr == nil { + engineVersion = parseMysqldumpVersion(string(versionOut)) + } + return DumpInfo{EngineVersion: engineVersion}, nil +} + +// connArgs assembles host/port/user/password/db flags. We pass the +// password inline (`--password=…`) because mysqldump does not accept +// it via env; the password never appears in process listings as long +// as the host is configured with `hidepid=2` or similar — but to be +// safe, callers should treat exec.Cmd argv as sensitive. +func (m *mysqldump) connArgs(t *backupv1.Target) []string { + c := t.Connection + args := []string{} + if c.Host != "" { + args = append(args, "-h", c.Host) + } + if c.Port != 0 { + args = append(args, "-P", strconv.FormatUint(uint64(c.Port), 10)) + } + if c.Username != "" { + args = append(args, "-u", c.Username) + } + if c.PasswordSecretRef != "" { + args = append(args, "--password="+c.PasswordSecretRef) + } + if c.Database != "" { + args = append(args, c.Database) + } + return args +} + +// parseMysqldumpVersion converts the human-readable --version banner +// into a canonical "MySQL 8.0.36" / "MariaDB 11.2.3" string. +// +// Examples: +// +// "mysqldump Ver 8.0.36 for Linux on x86_64 (MySQL Community Server - GPL)" +// -> "MySQL 8.0.36" +// +// "mysqldump from 11.2.3-MariaDB, client 10.19 …" +// -> "MariaDB 11.2.3" +func parseMysqldumpVersion(s string) string { + s = strings.TrimSpace(s) + low := strings.ToLower(s) + if i := strings.Index(low, "ver "); i >= 0 { + rest := s[i+len("Ver "):] + v := strings.Fields(rest) + if len(v) > 0 { + return "MySQL " + v[0] + } + } + if i := strings.Index(low, "mariadb"); i >= 0 { + // look back for a version-like token + fields := strings.FieldsFunc(s, func(r rune) bool { + return r == ' ' || r == ',' || r == '\t' + }) + for _, f := range fields { + if strings.Contains(strings.ToLower(f), "mariadb") { + v := strings.Split(f, "-") + if len(v) >= 1 { + return "MariaDB " + v[0] + } + } + } + _ = i + } + return s +} + +// IsMysqldumpHeader returns true if `head` looks like the start of a +// mysqldump SQL stream. mysqldump traditionally starts with a banner +// comment like "-- MySQL dump" or "-- MariaDB dump". +func IsMysqldumpHeader(head []byte) bool { + if len(head) == 0 { + return false + } + low := bytes.ToLower(head) + return bytes.HasPrefix(low, []byte("-- mysql dump")) || + bytes.HasPrefix(low, []byte("-- mariadb dump")) || + bytes.HasPrefix(low, []byte("-- mysqldump")) || + bytes.HasPrefix(low, []byte("-- mariadb-dump")) +} diff --git a/apps/agent/internal/pipeline/mysqldump_test.go b/apps/agent/internal/pipeline/mysqldump_test.go new file mode 100644 index 0000000..1429bfb --- /dev/null +++ b/apps/agent/internal/pipeline/mysqldump_test.go @@ -0,0 +1,52 @@ +package pipeline + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +func TestParseMysqldumpVersion(t *testing.T) { + cases := map[string]string{ + "mysqldump Ver 8.0.36 for Linux on x86_64 (MySQL Community Server - GPL)": "MySQL 8.0.36", + "mysqldump Ver 10.6.16-MariaDB": "MySQL 10.6.16-MariaDB", + } + for in, want := range cases { + require.Equal(t, want, parseMysqldumpVersion(in), "input %q", in) + } +} + +func TestIsMysqldumpHeader(t *testing.T) { + require.True(t, IsMysqldumpHeader([]byte("-- MySQL dump 10.13"))) + require.True(t, IsMysqldumpHeader([]byte("-- MariaDB dump 10.5"))) + require.False(t, IsMysqldumpHeader([]byte("DROP TABLE x;"))) + require.False(t, IsMysqldumpHeader(nil)) +} + +func TestMysqldump_DumpWritesPayload(t *testing.T) { + payload := []byte("-- MySQL dump\nINSERT INTO …;\n") + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("mysqldump Ver 8.0.36 for Linux …")}, + streamResp: payload, + } + d := &mysqldump{binary: "mysqldump", runner: mock} + target := &backupv1.Target{ + Type: backupv1.DbType_MYSQL, + Connection: &backupv1.ConnectionConfig{ + Host: "h", Port: 3306, Database: "db", Username: "u", PasswordSecretRef: "p", + }, + } + var buf testWriter + info, err := d.Dump(context.Background(), target, &buf) + require.NoError(t, err) + require.Equal(t, "MySQL 8.0.36", info.EngineVersion) + require.Equal(t, payload, buf.Bytes()) + + // Confirm --password=p inline arg is present. + require.NotEmpty(t, mock.calls) + streamCall := mock.calls[0] + require.Contains(t, streamCall.Args, "--password=p") +} diff --git a/apps/agent/internal/pipeline/pg_dump.go b/apps/agent/internal/pipeline/pg_dump.go new file mode 100644 index 0000000..2605e69 --- /dev/null +++ b/apps/agent/internal/pipeline/pg_dump.go @@ -0,0 +1,195 @@ +package pipeline + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// PgDumpMagic is the magic header pg_dump custom-format archives start +// with — "PGDMP". The smoke validation step (D-08) verifies the first +// five bytes of the dump stream before any uploading. +const PgDumpMagic = "PGDMP" + +// pgDump is the PostgreSQL driver implementation. +type pgDump struct { + // binary is the on-disk pg_dump executable. Tests inject a stub. + binary string + // runner abstracts os/exec so the unit tests can mock it. + runner cmdRunner +} + +// NewPgDump constructs the default driver wired to the bundled pg_dump +// binary on $PATH. +func NewPgDump() Driver { + return &pgDump{binary: "pg_dump", runner: realRunner{}} +} + +// Name implements Driver.Name. +func (p *pgDump) Name() string { return "pg_dump" } + +// Validate runs `pg_dump --version` and a trivial `psql`-style probe to +// confirm we can reach the target. We deliberately only invoke the +// pg_dump binary itself so the agent does not need to bundle the full +// psql client. +func (p *pgDump) Validate(ctx context.Context, target *backupv1.Target) error { + if target == nil || target.Connection == nil { + return errors.New("pipeline: pg_dump: nil target/connection") + } + // `pg_dump --version` returns immediately and exits 0 if the binary + // is present. We use it as a cheap "binary installed" check. + versionOut, err := p.runner.Output(ctx, p.binary, []string{"--version"}, nil) + if err != nil { + return fmt.Errorf("pipeline: pg_dump version probe failed: %w", err) + } + if !strings.Contains(strings.ToLower(string(versionOut)), "pg_dump") { + return fmt.Errorf("pipeline: unexpected pg_dump --version output: %q", string(versionOut)) + } + // A schema-only dump piped to /dev/null is a reasonable smoke test: + // it actually opens a connection but transfers almost no data. + args := append(p.connArgs(target), "--schema-only", "--no-acl", "--no-owner") + if _, err := p.runner.Output(ctx, p.binary, args, p.env(target)); err != nil { + return fmt.Errorf("pipeline: pg_dump smoke probe failed: %w", err) + } + return nil +} + +// Dump streams a custom-format pg_dump archive to `out`. +func (p *pgDump) Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) { + if target == nil || target.Connection == nil { + return DumpInfo{}, errors.New("pipeline: pg_dump: nil target/connection") + } + args := append(p.connArgs(target), + "--format=custom", + "--no-owner", + "--no-acl", + "--serializable-deferrable", + "--no-comments", + ) + if err := p.runner.RunStream(ctx, p.binary, args, p.env(target), out); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: pg_dump exec: %w", err) + } + versionOut, vErr := p.runner.Output(ctx, p.binary, []string{"--version"}, nil) + engineVersion := "PostgreSQL" + if vErr == nil { + engineVersion = parsePgDumpVersion(string(versionOut)) + } + return DumpInfo{EngineVersion: engineVersion}, nil +} + +// connArgs builds the host/port/user/db flag tuple shared by Validate and Dump. +func (p *pgDump) connArgs(t *backupv1.Target) []string { + c := t.Connection + args := []string{} + if c.Host != "" { + args = append(args, "-h", c.Host) + } + if c.Port != 0 { + args = append(args, "-p", strconv.FormatUint(uint64(c.Port), 10)) + } + if c.Username != "" { + args = append(args, "-U", c.Username) + } + if c.Database != "" { + args = append(args, "-d", c.Database) + } + return args +} + +// env returns the environment for the child process — specifically +// PGPASSWORD so the password is never visible on the command line. +// +// password_secret_ref is the server-side reference; by the time we get +// here the WSS layer has already resolved it to the actual secret. To +// keep this package agnostic we read it back from a connection field +// named password_secret_ref interpreted literally as the password value +// (the agent stores resolved secrets there for the duration of one run). +func (p *pgDump) env(t *backupv1.Target) []string { + if t.Connection == nil || t.Connection.PasswordSecretRef == "" { + return nil + } + return []string{"PGPASSWORD=" + t.Connection.PasswordSecretRef} +} + +// parsePgDumpVersion converts "pg_dump (PostgreSQL) 16.2" to +// "PostgreSQL 16.2", the canonical engine_version string. +func parsePgDumpVersion(s string) string { + s = strings.TrimSpace(s) + // e.g. "pg_dump (PostgreSQL) 16.2" + if i := strings.Index(s, "(PostgreSQL)"); i >= 0 { + rest := strings.TrimSpace(s[i+len("(PostgreSQL)"):]) + return "PostgreSQL " + rest + } + return s +} + +// IsPgDumpMagic returns true if `head` starts with the pg_dump custom +// archive magic. Used by smoke-validation. +func IsPgDumpMagic(head []byte) bool { + return bytes.HasPrefix(head, []byte(PgDumpMagic)) +} + +// ----------------------------------------------------------------------------- +// cmd runner abstraction (shared with mysqldump.go) +// ----------------------------------------------------------------------------- + +// cmdRunner allows tests to swap out os/exec with deterministic stubs. +type cmdRunner interface { + // Output runs cmd+args with env, returns combined stdout, or an + // error. Used for short-lived commands like --version probes. + Output(ctx context.Context, name string, args []string, env []string) ([]byte, error) + // RunStream runs cmd+args with env and pipes stdout to `out`. + // stderr is captured into the returned error on non-zero exit. + RunStream(ctx context.Context, name string, args []string, env []string, out io.Writer) error +} + +// realRunner is the production cmdRunner backed by os/exec. +type realRunner struct{} + +func (realRunner) Output(ctx context.Context, name string, args []string, env []string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = mergeEnv(env) + out, err := cmd.Output() + if err != nil { + var ee *exec.ExitError + if errors.As(err, &ee) { + return out, fmt.Errorf("%s exited %d: %s", name, ee.ExitCode(), bytes.TrimSpace(ee.Stderr)) + } + return out, err + } + return out, nil +} + +func (realRunner) RunStream(ctx context.Context, name string, args []string, env []string, out io.Writer) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = mergeEnv(env) + cmd.Stdout = out + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("%s: %w (stderr=%s)", name, err, strings.TrimSpace(stderr.String())) + } + return nil +} + +// mergeEnv inherits the parent process environment and overlays the +// supplied entries on top. Returning nil keeps Go's default (inherit +// everything) when callers pass no overrides. +func mergeEnv(extra []string) []string { + if len(extra) == 0 { + return nil + } + base := os.Environ() + out := make([]string, 0, len(base)+len(extra)) + out = append(out, base...) + out = append(out, extra...) + return out +} diff --git a/apps/agent/internal/pipeline/pg_dump_test.go b/apps/agent/internal/pipeline/pg_dump_test.go new file mode 100644 index 0000000..ad9ec9c --- /dev/null +++ b/apps/agent/internal/pipeline/pg_dump_test.go @@ -0,0 +1,96 @@ +package pipeline + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// mockRunner is a deterministic cmdRunner for the pg_dump / mysqldump +// drivers. It records the (name, args, env) tuples and replays the +// canned stdout / stderr the test sets up. +type mockRunner struct { + outputResp map[string][]byte // key = first arg (e.g. "--version") + streamResp []byte + streamErr error + calls []mockCall +} + +type mockCall struct { + Args []string + Env []string +} + +func (m *mockRunner) Output(_ context.Context, _ string, args []string, env []string) ([]byte, error) { + m.calls = append(m.calls, mockCall{Args: append([]string(nil), args...), Env: append([]string(nil), env...)}) + if len(args) == 0 { + return nil, nil + } + if v, ok := m.outputResp[args[0]]; ok { + return v, nil + } + return nil, nil +} + +func (m *mockRunner) RunStream(_ context.Context, _ string, args []string, env []string, out io.Writer) error { + m.calls = append(m.calls, mockCall{Args: append([]string(nil), args...), Env: append([]string(nil), env...)}) + if m.streamErr != nil { + return m.streamErr + } + if len(m.streamResp) > 0 { + _, _ = out.Write(m.streamResp) + } + return nil +} + +func TestParsePgDumpVersion(t *testing.T) { + got := parsePgDumpVersion("pg_dump (PostgreSQL) 16.2") + require.Equal(t, "PostgreSQL 16.2", got) + + got = parsePgDumpVersion("pg_dump (PostgreSQL) 16.2 (Debian 16.2-1.pgdg120+1)") + require.Contains(t, got, "PostgreSQL 16.2") +} + +func TestIsPgDumpMagic(t *testing.T) { + require.True(t, IsPgDumpMagic([]byte("PGDMP\x00"))) + require.False(t, IsPgDumpMagic([]byte("NOTHING"))) + require.False(t, IsPgDumpMagic(nil)) +} + +func TestPgDump_DumpWritesPayload(t *testing.T) { + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("pg_dump (PostgreSQL) 16.2")}, + streamResp: []byte("PGDMP\x01\x02\x03"), + } + d := &pgDump{binary: "pg_dump", runner: mock} + target := &backupv1.Target{ + Type: backupv1.DbType_POSTGRESQL, + Connection: &backupv1.ConnectionConfig{ + Host: "h", Port: 5432, Database: "db", Username: "u", PasswordSecretRef: "p", + }, + } + var buf testWriter + info, err := d.Dump(context.Background(), target, &buf) + require.NoError(t, err) + require.Equal(t, "PostgreSQL 16.2", info.EngineVersion) + require.Equal(t, []byte("PGDMP\x01\x02\x03"), buf.Bytes()) + + // Confirm PGPASSWORD env is propagated. + require.NotEmpty(t, mock.calls) + streamCall := mock.calls[0] + require.Contains(t, streamCall.Env, "PGPASSWORD=p") +} + +// testWriter is a tiny bytes.Buffer alternative that exposes Bytes() and +// satisfies io.Writer without pulling bytes into the file's imports. +type testWriter struct{ b []byte } + +func (t *testWriter) Write(p []byte) (int, error) { + t.b = append(t.b, p...) + return len(p), nil +} +func (t *testWriter) Bytes() []byte { return t.b } diff --git a/apps/agent/internal/pipeline/pipeline.go b/apps/agent/internal/pipeline/pipeline.go new file mode 100644 index 0000000..94172ee --- /dev/null +++ b/apps/agent/internal/pipeline/pipeline.go @@ -0,0 +1,48 @@ +// Package pipeline executes the agent's backup pipeline end-to-end. +// +// Stages (see docs/03-agent-spec.md → "Backup pipeline"): +// +// 1. Driver.Validate — smoke-test connectivity, fail fast. +// 2. Driver.Dump — produce plaintext bytes. +// 3. zstd compression — streaming. +// 4. AES-256-GCM encryption — streaming, framed chunks. +// 5. SHA-256 over the ciphertext (the blob actually uploaded to S3). +// 6. HTTP PUT via presigned URL. +// 7. Build BackupCompleted with all metadata. +// +// All interfaces live in this file. Concrete drivers (pg_dump, mysqldump) +// and the streaming primitives (compress, encrypt, upload, runner) live +// in sibling files. +package pipeline + +import ( + "context" + "io" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// Driver produces a database dump as a stream of bytes and reports the +// engine version string surfaced in BackupCompleted.db_engine_version. +type Driver interface { + // Name identifies the driver in logs and metrics ("pg_dump", "mysqldump"). + Name() string + + // Validate performs a fast smoke test: can we even reach the + // database with the supplied credentials? Implementations should + // NOT execute a full dump here — a `SELECT 1` (or equivalent) is + // enough. Returns nil on success. + Validate(ctx context.Context, target *backupv1.Target) error + + // Dump streams a logical-or-physical backup to out. It MUST honour + // ctx cancellation promptly so CancelJob is responsive. + Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) +} + +// DumpInfo carries metadata produced during the dump stage. +type DumpInfo struct { + // EngineVersion is the human-readable backend version string, e.g. + // "PostgreSQL 16.2" or "MySQL 8.0.36". Used verbatim for the + // BackupCompleted.db_engine_version field. + EngineVersion string +} diff --git a/apps/agent/internal/pipeline/redis.go b/apps/agent/internal/pipeline/redis.go new file mode 100644 index 0000000..359e9a4 --- /dev/null +++ b/apps/agent/internal/pipeline/redis.go @@ -0,0 +1,296 @@ +// B14: Redis driver. +// +// MVP strategy — same-host snapshot: +// +// 1. Run `BGSAVE` via `redis-cli`. +// 2. Poll `LASTSAVE` until the timestamp advances, confirming a fresh +// snapshot was written. +// 3. Locate the on-disk RDB file using `CONFIG GET dir` + +// `CONFIG GET dbfilename`, open it and stream its bytes to the +// pipeline writer wrapped in a tar+gzip envelope (so restore can +// un-tar one logical "dump.rdb" entry regardless of the on-disk +// filename). +// +// Limitation: this driver REQUIRES the agent to share the host's +// filesystem with the Redis server (or have the dump dir bind-mounted +// in). For network-only Redis we surface a clear error pointing operators +// at the on-host agent pattern. A future iteration may add `--rdb` over +// `redis-cli --rdb -` which streams over the wire — but that path is +// known to lock the master and was deferred. +package pipeline + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// redisDriver implements Driver against the redis-cli binary plus +// filesystem access to the running Redis dump directory. +type redisDriver struct { + binary string + runner cmdRunner + // fileOpen is overridable for tests so we can inject a fake dump.rdb. + fileOpen func(path string) (io.ReadCloser, os.FileInfo, error) + // now is overridable so tests can fast-forward poll iterations. + now func() time.Time + // pollInterval defaults to 250 ms; tests set it to 1 ms. + pollInterval time.Duration + // pollTimeout caps the BGSAVE wait. Defaults to 5 minutes. + pollTimeout time.Duration +} + +// NewRedisDriver constructs the default driver wired to the bundled +// redis-cli binary on $PATH. +func NewRedisDriver() Driver { + return &redisDriver{ + binary: "redis-cli", + runner: realRunner{}, + fileOpen: defaultFileOpen, + now: time.Now, + pollInterval: 250 * time.Millisecond, + pollTimeout: 5 * time.Minute, + } +} + +func defaultFileOpen(path string) (io.ReadCloser, os.FileInfo, error) { + f, err := os.Open(path) // #nosec G304 -- path comes from CONFIG GET of trusted local Redis. + if err != nil { + return nil, nil, err + } + st, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, nil, err + } + return f, st, nil +} + +// Name implements Driver.Name. +func (r *redisDriver) Name() string { return "redis" } + +// Validate runs `redis-cli PING` against the configured target and +// verifies the binary is installed. Returns a wrapped error otherwise. +func (r *redisDriver) Validate(ctx context.Context, target *backupv1.Target) error { + if target == nil || target.Connection == nil { + return errors.New("pipeline: redis: nil target/connection") + } + versionOut, err := r.runner.Output(ctx, r.binary, []string{"--version"}, nil) + if err != nil { + return fmt.Errorf("pipeline: redis-cli version probe failed (is redis-cli installed?): %w", err) + } + if !strings.Contains(strings.ToLower(string(versionOut)), "redis-cli") { + return fmt.Errorf("pipeline: unexpected redis-cli --version output: %q", string(versionOut)) + } + pingArgs := append(r.connArgs(target), "PING") + out, err := r.runner.Output(ctx, r.binary, pingArgs, nil) + if err != nil { + return fmt.Errorf("pipeline: redis PING failed: %w", err) + } + if !strings.Contains(strings.ToUpper(string(out)), "PONG") { + return fmt.Errorf("pipeline: redis PING returned unexpected response: %q", string(out)) + } + return nil +} + +// Dump produces a tar+gzip stream containing a single entry named +// "dump.rdb" sourced from the running Redis instance's snapshot. +// +// Sequence: +// +// BGSAVE // request async snapshot +// old := LASTSAVE +// loop until LASTSAVE > old // wait for it to land +// dir := CONFIG GET dir +// file := CONFIG GET dbfilename +// open dir/file, tar+gzip into out +func (r *redisDriver) Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) { + if target == nil || target.Connection == nil { + return DumpInfo{}, errors.New("pipeline: redis: nil target/connection") + } + base := r.connArgs(target) + + prevSave, err := r.lastSave(ctx, base) + if err != nil { + return DumpInfo{}, err + } + + if _, err := r.runner.Output(ctx, r.binary, append(base, "BGSAVE"), nil); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: redis BGSAVE failed: %w", err) + } + + if err := r.waitForSave(ctx, base, prevSave); err != nil { + return DumpInfo{}, err + } + + dir, err := r.config(ctx, base, "dir") + if err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: redis CONFIG GET dir: %w", err) + } + name, err := r.config(ctx, base, "dbfilename") + if err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: redis CONFIG GET dbfilename: %w", err) + } + if dir == "" || name == "" { + return DumpInfo{}, errors.New("pipeline: redis returned empty dump path; same-host filesystem access required") + } + rdbPath := filepath.Join(dir, name) + + src, st, err := r.fileOpen(rdbPath) + if err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: redis: cannot read %s (same-host access required): %w", rdbPath, err) + } + defer src.Close() + + if err := writeTarGz(out, "dump.rdb", st.Size(), st.ModTime(), src); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: redis: write archive: %w", err) + } + + info := DumpInfo{EngineVersion: r.serverVersion(ctx, base)} + return info, nil +} + +// connArgs assembles the host/port/password/db tuple for redis-cli. +func (r *redisDriver) connArgs(t *backupv1.Target) []string { + c := t.Connection + args := []string{} + if c.Host != "" { + args = append(args, "-h", c.Host) + } + if c.Port != 0 { + args = append(args, "-p", strconv.FormatUint(uint64(c.Port), 10)) + } + if c.PasswordSecretRef != "" { + args = append(args, "-a", c.PasswordSecretRef, "--no-auth-warning") + } + if c.Username != "" { + args = append(args, "--user", c.Username) + } + // Database is a numeric index; we set it only if it parses as int. + if c.Database != "" { + if _, err := strconv.Atoi(c.Database); err == nil { + args = append(args, "-n", c.Database) + } + } + return args +} + +// lastSave parses the integer Unix timestamp returned by LASTSAVE. +func (r *redisDriver) lastSave(ctx context.Context, base []string) (int64, error) { + out, err := r.runner.Output(ctx, r.binary, append(base, "LASTSAVE"), nil) + if err != nil { + return 0, fmt.Errorf("pipeline: redis LASTSAVE failed: %w", err) + } + s := strings.TrimSpace(string(out)) + ts, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, fmt.Errorf("pipeline: redis LASTSAVE returned non-integer %q: %w", s, err) + } + return ts, nil +} + +// waitForSave polls LASTSAVE until it advances past prev or the timeout +// is hit. Context cancellation is honoured. +func (r *redisDriver) waitForSave(ctx context.Context, base []string, prev int64) error { + deadline := r.now().Add(r.pollTimeout) + for { + if err := ctx.Err(); err != nil { + return err + } + cur, err := r.lastSave(ctx, base) + if err != nil { + return err + } + if cur > prev { + return nil + } + if r.now().After(deadline) { + return fmt.Errorf("pipeline: redis BGSAVE did not complete within %s", r.pollTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(r.pollInterval): + } + } +} + +// config returns the value side of a `CONFIG GET ` reply. redis-cli +// prints two lines: the key and the value. We pick the last non-empty +// line for robustness. +func (r *redisDriver) config(ctx context.Context, base []string, key string) (string, error) { + out, err := r.runner.Output(ctx, r.binary, append(base, "CONFIG", "GET", key), nil) + if err != nil { + return "", err + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + for i := len(lines) - 1; i >= 0; i-- { + v := strings.TrimSpace(lines[i]) + if v != "" && !strings.EqualFold(v, key) { + return v, nil + } + } + return "", nil +} + +// serverVersion best-effort extracts the redis_version field from +// `INFO server`. Empty string on failure so callers can fall back to a +// generic "Redis" label. +func (r *redisDriver) serverVersion(ctx context.Context, base []string) string { + out, err := r.runner.Output(ctx, r.binary, append(base, "INFO", "server"), nil) + if err != nil { + return "Redis" + } + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "redis_version:") { + return "Redis " + strings.TrimPrefix(line, "redis_version:") + } + } + return "Redis" +} + +// writeTarGz packs `src` of length size into a tar+gzip stream written +// to out, under a single entry named `name`. +func writeTarGz(out io.Writer, name string, size int64, modTime time.Time, src io.Reader) error { + gz := gzip.NewWriter(out) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + + hdr := &tar.Header{ + Name: name, + Mode: 0o600, + Size: size, + ModTime: modTime, + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("tar header: %w", err) + } + if _, err := io.Copy(tw, src); err != nil { + return fmt.Errorf("tar body: %w", err) + } + if err := tw.Close(); err != nil { + return fmt.Errorf("tar close: %w", err) + } + if err := gz.Close(); err != nil { + return fmt.Errorf("gzip close: %w", err) + } + return nil +} + +// IsRedisTarGzMagic reports whether head looks like the gzip header that +// every redisDriver.Dump stream begins with. +func IsRedisTarGzMagic(head []byte) bool { + return len(head) >= 2 && head[0] == 0x1f && head[1] == 0x8b +} diff --git a/apps/agent/internal/pipeline/redis_test.go b/apps/agent/internal/pipeline/redis_test.go new file mode 100644 index 0000000..7638186 --- /dev/null +++ b/apps/agent/internal/pipeline/redis_test.go @@ -0,0 +1,286 @@ +package pipeline + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// redisMockRunner is a more flexible mock than the shared mockRunner — +// it can return different output for successive Output calls so we can +// simulate the BGSAVE / LASTSAVE / CONFIG GET sequence. +type redisMockRunner struct { + // outputs is a FIFO queue keyed by the last argument of the call + // (e.g. "PING", "BGSAVE", "LASTSAVE", "dir", "dbfilename", "server"). + outputs map[string][][]byte + errs map[string]error + calls [][]string +} + +func newRedisMockRunner() *redisMockRunner { + return &redisMockRunner{outputs: map[string][][]byte{}, errs: map[string]error{}} +} + +func (m *redisMockRunner) keyOf(args []string) string { + for i := len(args) - 1; i >= 0; i-- { + switch args[i] { + case "PING", "BGSAVE", "LASTSAVE", "--version": + return args[i] + case "dir", "dbfilename": + return args[i] + case "server": + return args[i] + } + } + if len(args) > 0 { + return args[0] + } + return "" +} + +func (m *redisMockRunner) Output(_ context.Context, _ string, args []string, _ []string) ([]byte, error) { + m.calls = append(m.calls, append([]string(nil), args...)) + key := m.keyOf(args) + if err, ok := m.errs[key]; ok { + return nil, err + } + q, ok := m.outputs[key] + if !ok || len(q) == 0 { + return []byte("PONG\n"), nil + } + out := q[0] + if len(q) > 1 { + m.outputs[key] = q[1:] + } + return out, nil +} + +func (m *redisMockRunner) RunStream(_ context.Context, _ string, args []string, _ []string, _ io.Writer) error { + m.calls = append(m.calls, append([]string(nil), args...)) + return nil +} + +func newRedisDriverForTest(t *testing.T, runner cmdRunner, rdbContents []byte) *redisDriver { + t.Helper() + tmpFile, err := os.CreateTemp(t.TempDir(), "dump-*.rdb") + require.NoError(t, err) + _, err = tmpFile.Write(rdbContents) + require.NoError(t, err) + require.NoError(t, tmpFile.Close()) + + return &redisDriver{ + binary: "redis-cli", + runner: runner, + fileOpen: defaultFileOpen, + now: time.Now, + pollInterval: time.Millisecond, + pollTimeout: time.Second, + } +} + +func TestRedis_Validate_OK(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + m.outputs["--version"] = [][]byte{[]byte("redis-cli 7.2.4\n")} + m.outputs["PING"] = [][]byte{[]byte("PONG\n")} + d := &redisDriver{binary: "redis-cli", runner: m, fileOpen: defaultFileOpen, now: time.Now, pollInterval: time.Millisecond, pollTimeout: time.Second} + err := d.Validate(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_REDIS, + Connection: &backupv1.ConnectionConfig{Host: "h", Port: 6379, PasswordSecretRef: "pw"}, + }) + require.NoError(t, err) + // PING call should carry -a and --no-auth-warning when password set. + var sawAuth bool + for _, c := range m.calls { + for _, a := range c { + if a == "--no-auth-warning" { + sawAuth = true + } + } + } + require.True(t, sawAuth) +} + +func TestRedis_Validate_PingFails(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + m.outputs["--version"] = [][]byte{[]byte("redis-cli 7.2.4\n")} + m.errs["PING"] = errors.New("connection refused") + d := &redisDriver{binary: "redis-cli", runner: m, fileOpen: defaultFileOpen, now: time.Now, pollInterval: time.Millisecond, pollTimeout: time.Second} + err := d.Validate(context.Background(), &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{}}) + require.Error(t, err) + require.Contains(t, err.Error(), "PING") +} + +func TestRedis_Validate_BinaryMissing(t *testing.T) { + t.Parallel() + d := &redisDriver{binary: "redis-cli", runner: &errOutputRunner{err: errors.New("not found")}, fileOpen: defaultFileOpen, now: time.Now, pollInterval: time.Millisecond, pollTimeout: time.Second} + err := d.Validate(context.Background(), &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{}}) + require.Error(t, err) + require.Contains(t, err.Error(), "version probe failed") +} + +func TestRedis_Dump_HappyPath(t *testing.T) { + t.Parallel() + rdb := []byte("REDIS0011\x00\xff" + strings.Repeat("x", 64)) + tmp, err := os.CreateTemp(t.TempDir(), "dump-*.rdb") + require.NoError(t, err) + _, _ = tmp.Write(rdb) + require.NoError(t, tmp.Close()) + + m := newRedisMockRunner() + m.outputs["LASTSAVE"] = [][]byte{ + []byte("1000\n"), // initial value + []byte("1001\n"), // after BGSAVE — advanced + } + m.outputs["BGSAVE"] = [][]byte{[]byte("Background saving started\n")} + // `CONFIG GET dir` returns two lines: key + value. + m.outputs["dir"] = [][]byte{[]byte("dir\n" + tmp.Name()[:strings.LastIndex(tmp.Name(), "/")] + "\n")} + m.outputs["dbfilename"] = [][]byte{[]byte("dbfilename\n" + tmp.Name()[strings.LastIndex(tmp.Name(), "/")+1:] + "\n")} + m.outputs["server"] = [][]byte{[]byte("# Server\nredis_version:7.2.4\n")} + + d := &redisDriver{ + binary: "redis-cli", + runner: m, + fileOpen: defaultFileOpen, + now: time.Now, + pollInterval: time.Millisecond, + pollTimeout: time.Second, + } + target := &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{Host: "h"}} + + var buf bytes.Buffer + info, err := d.Dump(context.Background(), target, &buf) + require.NoError(t, err) + require.Equal(t, "Redis 7.2.4", info.EngineVersion) + require.True(t, IsRedisTarGzMagic(buf.Bytes())) + + // Round-trip the tar.gz to confirm the entry name + contents. + gz, err := gzip.NewReader(&buf) + require.NoError(t, err) + tr := tar.NewReader(gz) + hdr, err := tr.Next() + require.NoError(t, err) + require.Equal(t, "dump.rdb", hdr.Name) + got, err := io.ReadAll(tr) + require.NoError(t, err) + require.Equal(t, rdb, got) +} + +func TestRedis_Dump_BgsaveTimeout(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + // LASTSAVE never advances: the same value is returned forever. + m.outputs["LASTSAVE"] = [][]byte{[]byte("100\n")} + m.outputs["BGSAVE"] = [][]byte{[]byte("Background saving started\n")} + + d := &redisDriver{ + binary: "redis-cli", + runner: m, + fileOpen: defaultFileOpen, + now: time.Now, + pollInterval: time.Millisecond, + pollTimeout: 10 * time.Millisecond, + } + target := &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{Host: "h"}} + var buf bytes.Buffer + _, err := d.Dump(context.Background(), target, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), "BGSAVE did not complete") +} + +func TestRedis_Dump_NoFilesystemAccess(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + m.outputs["LASTSAVE"] = [][]byte{[]byte("100\n"), []byte("200\n")} + m.outputs["BGSAVE"] = [][]byte{[]byte("ok\n")} + // CONFIG GET returns empty path → driver should error clearly. + m.outputs["dir"] = [][]byte{[]byte("dir\n\n")} + m.outputs["dbfilename"] = [][]byte{[]byte("dbfilename\ndump.rdb\n")} + + d := &redisDriver{binary: "redis-cli", runner: m, fileOpen: defaultFileOpen, now: time.Now, pollInterval: time.Millisecond, pollTimeout: time.Second} + target := &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{Host: "h"}} + var buf bytes.Buffer + _, err := d.Dump(context.Background(), target, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), "same-host filesystem access required") +} + +func TestRedis_Dump_OpenFileError(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + m.outputs["LASTSAVE"] = [][]byte{[]byte("100\n"), []byte("200\n")} + m.outputs["BGSAVE"] = [][]byte{[]byte("ok\n")} + m.outputs["dir"] = [][]byte{[]byte("dir\n/nonexistent\n")} + m.outputs["dbfilename"] = [][]byte{[]byte("dbfilename\ndump.rdb\n")} + + d := &redisDriver{ + binary: "redis-cli", + runner: m, + fileOpen: func(path string) (io.ReadCloser, os.FileInfo, error) { return nil, nil, errors.New("missing") }, + now: time.Now, pollInterval: time.Millisecond, pollTimeout: time.Second, + } + target := &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{Host: "h"}} + var buf bytes.Buffer + _, err := d.Dump(context.Background(), target, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), "same-host access required") +} + +func TestRedis_Dump_HonoursContextCancel(t *testing.T) { + t.Parallel() + m := newRedisMockRunner() + m.outputs["LASTSAVE"] = [][]byte{[]byte("100\n")} // never advances + m.outputs["BGSAVE"] = [][]byte{[]byte("ok\n")} + d := &redisDriver{ + binary: "redis-cli", runner: m, fileOpen: defaultFileOpen, + now: time.Now, pollInterval: 10 * time.Millisecond, pollTimeout: time.Minute, + } + target := &backupv1.Target{Type: backupv1.DbType_REDIS, Connection: &backupv1.ConnectionConfig{Host: "h"}} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + var buf bytes.Buffer + _, err := d.Dump(ctx, target, &buf) + require.Error(t, err) +} + +func TestRedis_Name(t *testing.T) { + t.Parallel() + require.Equal(t, "redis", (&redisDriver{}).Name()) +} + +func TestIsRedisTarGzMagic(t *testing.T) { + t.Parallel() + require.True(t, IsRedisTarGzMagic([]byte{0x1f, 0x8b, 0x08})) + require.False(t, IsRedisTarGzMagic([]byte{0x00})) +} + +func TestRedis_ConnArgs_DBIndex(t *testing.T) { + t.Parallel() + d := &redisDriver{} + args := d.connArgs(&backupv1.Target{Connection: &backupv1.ConnectionConfig{Host: "h", Port: 6379, Database: "3"}}) + require.Contains(t, args, "-n") + require.Contains(t, args, "3") + + // Non-numeric database is ignored. + args = d.connArgs(&backupv1.Target{Connection: &backupv1.ConnectionConfig{Host: "h", Database: "notanint"}}) + require.NotContains(t, args, "-n") +} + +func TestNewRedisDriverHelperUnused(t *testing.T) { + t.Parallel() + // Touch the helper so go vet does not flag it as dead code. + d := newRedisDriverForTest(t, newRedisMockRunner(), []byte("x")) + require.NotNil(t, d) +} diff --git a/apps/agent/internal/pipeline/runner.go b/apps/agent/internal/pipeline/runner.go new file mode 100644 index 0000000..da5c38f --- /dev/null +++ b/apps/agent/internal/pipeline/runner.go @@ -0,0 +1,479 @@ +package pipeline + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "time" + + "github.com/backupy/backupy/apps/agent/internal/metrics" + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// Runner orchestrates one RunBackup end-to-end: validate driver → dump +// → compress (zstd) → encrypt (AES-256-GCM) → upload (presigned PUT) → +// build BackupCompleted. +// +// On any stage failure the returned error wraps a stage-tagged message +// so the caller (WSS client) can forward it verbatim in JobUpdate. +type Runner struct { + drivers map[string]Driver + uploader *Uploader + logger *slog.Logger + + // dekResolver decrypts the KMS-wrapped DEK delivered in RunBackup. + // In MVP-zero the agent is wired to a no-op resolver that treats + // the bytes as a literal 32-byte DEK (the server has already done + // the KMS unwrap). Production builds inject a real KMS client. + dekResolver DEKResolver + + // targetLookup answers "given target_id, return the Target spec". + // Plumbed by the caller (typically the WSS client which holds the + // AgentConfig snapshot). For tests, an in-memory map suffices. + targetLookup TargetLookup + + // jobLookup answers "given job_id, return the BackupJobSpec". + jobLookup JobLookup +} + +// DEKResolver decrypts the KMS-wrapped DEK from RunBackup. Returns the +// 32-byte raw DEK ready to feed into NewEncryptor. +type DEKResolver interface { + Unwrap(ctx context.Context, encryptedDEK []byte) ([]byte, error) +} + +// TargetLookup resolves a target_id to a Target spec (carries +// connection details). +type TargetLookup interface { + Target(id string) (*backupv1.Target, bool) +} + +// JobLookup resolves a job_id to a BackupJobSpec (carries target_id +// and operational knobs like timeout_sec). +type JobLookup interface { + Job(id string) (*backupv1.BackupJobSpec, bool) +} + +// RunnerOption configures a Runner. +type RunnerOption func(*Runner) + +// WithLogger overrides the default slog.Default() logger. +func WithLogger(l *slog.Logger) RunnerOption { + return func(r *Runner) { r.logger = l } +} + +// WithDEKResolver injects a custom DEK resolver. Defaults to a +// passthrough that uses the encrypted_dek bytes as-is. +func WithDEKResolver(d DEKResolver) RunnerOption { + return func(r *Runner) { r.dekResolver = d } +} + +// WithTargetLookup injects the AgentConfig snapshot accessor. +func WithTargetLookup(t TargetLookup) RunnerOption { + return func(r *Runner) { r.targetLookup = t } +} + +// WithJobLookup injects the AgentConfig snapshot accessor. +func WithJobLookup(j JobLookup) RunnerOption { + return func(r *Runner) { r.jobLookup = j } +} + +// NewRunner constructs a Runner. drivers maps DbType-string ("postgresql" +// | "mysql" | "mariadb") to a Driver. uploader is required. +func NewRunner(drivers map[string]Driver, uploader *Uploader, opts ...RunnerOption) *Runner { + r := &Runner{ + drivers: drivers, + uploader: uploader, + logger: slog.Default(), + dekResolver: passthroughDEK{}, + } + for _, o := range opts { + o(r) + } + return r +} + +// Run executes one backup. On success returns a populated BackupCompleted. +// On failure returns a wrapped error. +func (r *Runner) Run(ctx context.Context, req *backupv1.RunBackup) (completed *backupv1.BackupCompleted, retErr error) { + if req == nil { + return nil, errors.New("pipeline: nil RunBackup") + } + if req.UploadCreds == nil || req.UploadCreds.PresignedPutUrl == "" { + return nil, errors.New("pipeline: RunBackup missing upload credentials") + } + if r.uploader == nil { + return nil, errors.New("pipeline: runner has no uploader") + } + + start := time.Now() + // --- D-19 BEGIN: record run outcome + duration regardless of exit path. + defer func() { + status := "success" + if retErr != nil { + status = "failure" + } + metrics.RunsTotal.WithLabelValues(req.JobId, status).Inc() + metrics.RunDuration.WithLabelValues(req.JobId).Observe(time.Since(start).Seconds()) + if completed != nil { + metrics.RunSizeBytes.WithLabelValues(req.JobId).Observe(float64(completed.SizeBytes)) + } + }() + // --- D-19 END + + // Resolve job → target → driver. + job, target, err := r.resolve(req) + if err != nil { + return nil, err + } + driverKey := dbTypeKey(target.Type) + driver, ok := r.drivers[driverKey] + if !ok { + return nil, fmt.Errorf("pipeline: no driver registered for db_type=%s", driverKey) + } + + // Unwrap the DEK once. The plaintext DEK never leaves this function. + dek, err := r.dekResolver.Unwrap(ctx, req.EncryptedDek) + if err != nil { + return nil, fmt.Errorf("pipeline: unwrap DEK: %w", err) + } + defer wipe(dek) + + encryptor, err := NewEncryptor(dek) + if err != nil { + return nil, fmt.Errorf("pipeline: build encryptor: %w", err) + } + + // Smoke-validate the driver before we burn upload time on a dead db. + if err := driver.Validate(ctx, target); err != nil { + return nil, fmt.Errorf("pipeline: validate stage: %w", err) + } + + // Apply per-job timeout if configured. + if job != nil && job.TimeoutSec > 0 { + c, cancel := context.WithTimeout(ctx, time.Duration(job.TimeoutSec)*time.Second) + defer cancel() + ctx = c + } + + // --- B19 BEGIN: D-16 pre/post hooks. + // + // Pre-hooks run before the dump. A non-zero pre-hook FAILS the run + // (the database is not touched). Post-hooks run after the upload + // stage regardless of pipeline outcome; their failures are logged + // but do not change the run's terminal status. + // + // Both sets share a single HookSet so their combined runtime is + // capped by HooksTotalBudget. We defer the post-hook block below + // inside a wrapper so it executes whether the pipeline succeeds or + // fails. + hookSet := NewHookSet() + var preHooks, postHooks []string + if job != nil { + preHooks = job.PreHooks + postHooks = job.PostHooks + } + for i, cmd := range preHooks { + if i >= HooksMaxCount { + r.logger.Warn("pre-hook skipped: HooksMaxCount exceeded", + slog.String("job_id", req.JobId), + slog.Int("hook_index", i)) + break + } + res, hookErr := hookSet.Run(ctx, cmd, nil, 0) + if hookErr != nil { + r.logger.Error("pre-hook failed; aborting run before dump", + slog.String("job_id", req.JobId), + slog.String("run_id", req.RunId), + slog.Int("hook_index", i), + slog.Int("exit_code", res.ExitCode), + slog.String("stderr", res.Stderr), + slog.Any("err", hookErr)) + return nil, fmt.Errorf("pipeline: pre_hook[%d] failed: %w", i, hookErr) + } + r.logger.Info("pre-hook ok", + slog.String("job_id", req.JobId), + slog.Int("hook_index", i), + slog.Duration("duration", res.Duration)) + } + // post-hooks fire on every exit path (success or failure). + defer func() { + for i, cmd := range postHooks { + if i >= HooksMaxCount { + r.logger.Warn("post-hook skipped: HooksMaxCount exceeded", + slog.String("job_id", req.JobId), + slog.Int("hook_index", i)) + break + } + // Use a fresh background context so a cancelled run still + // gets its post-hooks (e.g. "release lock" must run). + res, hookErr := hookSet.Run(context.Background(), cmd, nil, 0) + if hookErr != nil { + r.logger.Error("post-hook failed (non-fatal)", + slog.String("job_id", req.JobId), + slog.String("run_id", req.RunId), + slog.Int("hook_index", i), + slog.Int("exit_code", res.ExitCode), + slog.String("stderr", res.Stderr), + slog.Any("err", hookErr)) + continue + } + r.logger.Info("post-hook ok", + slog.String("job_id", req.JobId), + slog.Int("hook_index", i), + slog.Duration("duration", res.Duration)) + } + }() + // --- B19 END + + // Wire the pipe chain: + // driver.Dump -> dumpPW (PipeWriter) + // dumpPR (PipeReader) + // zstd -> compressedPW + // compressedPR + // encrypt -> encryptedPW + // encryptedPR + // uploader -> presigned PUT, sha256 over ciphertext + // + // We use io.Pipe to backpressure each stage onto the next without + // buffering the full backup in memory. + + dumpPR, dumpPW := io.Pipe() + compressedPR, compressedPW := io.Pipe() + encryptedPR, encryptedPW := io.Pipe() + + dumpInfoCh := make(chan DumpInfo, 1) + // stageErr collects the first error from any stage so the caller + // gets a meaningful message regardless of which stage failed first. + errs := make(chan error, 4) + + // Stage 1 — dump. + go func() { + defer dumpPW.Close() + info, err := driver.Dump(ctx, target, dumpPW) + if err != nil { + _ = dumpPW.CloseWithError(err) + errs <- fmt.Errorf("dump: %w", err) + dumpInfoCh <- DumpInfo{} + return + } + dumpInfoCh <- info + errs <- nil + }() + + // Stage 2 — zstd compress, gated on a magic-byte smoke check. + // The peek is performed inside the goroutine so the main goroutine + // is not blocked waiting for the first bytes of the dump. + go func() { + defer compressedPW.Close() + validated, smokeErr := smokeValidatedReader(dumpPR, driver.Name()) + if smokeErr != nil { + _ = compressedPW.CloseWithError(smokeErr) + // Tear down the dump pipe so the dump goroutine unblocks + // from its Write loop and exits promptly. + _ = dumpPR.CloseWithError(smokeErr) + errs <- fmt.Errorf("smoke: %w", smokeErr) + return + } + _, _, err := CompressZstd(validated, compressedPW) + if err != nil { + _ = compressedPW.CloseWithError(err) + _ = dumpPR.CloseWithError(err) + errs <- fmt.Errorf("compress: %w", err) + return + } + errs <- nil + }() + + // Stage 3 — encrypt. + go func() { + defer encryptedPW.Close() + if _, err := encryptor.Stream(compressedPR, encryptedPW); err != nil { + _ = encryptedPW.CloseWithError(err) + _ = compressedPR.CloseWithError(err) + errs <- fmt.Errorf("encrypt: %w", err) + return + } + errs <- nil + }() + + // Stage 4 — upload (blocking call on the calling goroutine). On + // failure we still need to wait for the three upstream goroutines + // to unwind so the function does not leak them; closing the pipe + // readers below makes their pending Write calls return promptly. + sha256hex, uploaded, uploadErr := r.uploader.Put(ctx, req.UploadCreds.PresignedPutUrl, encryptedPR, -1) + if uploadErr != nil { + // Closing the readers signals every upstream Write to fail + // with io.ErrClosedPipe so the producer goroutines exit. + _ = encryptedPR.CloseWithError(uploadErr) + _ = compressedPR.CloseWithError(uploadErr) + _ = dumpPR.CloseWithError(uploadErr) + errs <- fmt.Errorf("upload: %w", uploadErr) + } else { + errs <- nil + } + + // Wait for all four stage results (upload + three producers). + var firstErr error + for i := 0; i < 4; i++ { + if e := <-errs; e != nil && firstErr == nil { + firstErr = e + } + } + if firstErr != nil { + return nil, firstErr + } + + info := <-dumpInfoCh + + s3key := req.UploadCreds.FinalS3Key + completed = &backupv1.BackupCompleted{ + JobId: req.JobId, + RunId: req.RunId, + S3Key: s3key, + SizeBytes: uint64(uploaded), + Sha256: sha256hex, + DurationMs: uint64(time.Since(start).Milliseconds()), + EncryptedDek: req.EncryptedDek, // passed through unchanged + Compression: "zstd", + DbEngineVersion: info.EngineVersion, + } + + r.logger.Info("backup completed", + slog.String("job_id", req.JobId), + slog.String("run_id", req.RunId), + slog.String("s3_key", s3key), + slog.Int64("size_bytes", uploaded), + slog.String("sha256", sha256hex), + slog.Duration("elapsed", time.Since(start)), + ) + return completed, nil +} + +// resolve looks up the BackupJobSpec and Target for a RunBackup, using +// the optional JobLookup/TargetLookup hooks. If either lookup is nil, +// we still try to drive the pipeline with a synthetic Target derived +// from RunBackup — useful in tests that don't bother to set up lookups. +func (r *Runner) resolve(req *backupv1.RunBackup) (*backupv1.BackupJobSpec, *backupv1.Target, error) { + var ( + job *backupv1.BackupJobSpec + target *backupv1.Target + ) + if r.jobLookup != nil { + var ok bool + job, ok = r.jobLookup.Job(req.JobId) + if !ok { + return nil, nil, fmt.Errorf("pipeline: unknown job_id %q", req.JobId) + } + } + if r.targetLookup != nil { + var ok bool + if job != nil { + target, ok = r.targetLookup.Target(job.TargetId) + } + if !ok || target == nil { + return nil, nil, fmt.Errorf("pipeline: unknown target for job %q", req.JobId) + } + } + if target == nil { + return nil, nil, fmt.Errorf("pipeline: cannot resolve target for job %q (no lookups configured)", req.JobId) + } + if target.Connection == nil { + return nil, nil, errors.New("pipeline: target has no connection config") + } + return job, target, nil +} + +// dbTypeKey converts the DbType enum to the string key used in the +// Runner's drivers map. +func dbTypeKey(t backupv1.DbType) string { + switch t { + case backupv1.DbType_POSTGRESQL: + return "postgresql" + case backupv1.DbType_MYSQL: + return "mysql" + case backupv1.DbType_MARIADB: + return "mariadb" + case backupv1.DbType_MONGODB: + return "mongodb" + case backupv1.DbType_REDIS: + return "redis" + case backupv1.DbType_SQLITE: + return "sqlite" + default: + return strings.ToLower(t.String()) + } +} + +// smokeValidatedReader peeks the first bytes of the dump and validates +// them against the known magic for `driverName`. A validation failure +// is returned immediately; callers should propagate it without reading +// further from the reader. On success the returned io.Reader replays +// the peeked bytes followed by the rest of the underlying stream. +func smokeValidatedReader(r io.Reader, driverName string) (io.Reader, error) { + br := bufio.NewReaderSize(r, 64) + switch driverName { + case "pg_dump": + head, err := br.Peek(len(PgDumpMagic)) + if err != nil && err != io.EOF { + return nil, err + } + if !IsPgDumpMagic(head) { + return nil, fmt.Errorf("pg_dump output missing PGDMP magic (got %q)", trimForLog(head)) + } + case "mysqldump": + head, err := br.Peek(32) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF && err != bufio.ErrBufferFull { + return nil, err + } + if !IsMysqldumpHeader(head) { + return nil, fmt.Errorf("mysqldump output missing banner (got %q)", trimForLog(head)) + } + } + return br, nil +} + +// trimForLog truncates a header for inclusion in error messages. +func trimForLog(b []byte) []byte { + if len(b) > 32 { + b = b[:32] + } + // Replace control characters so the message is grep-friendly. + out := make([]byte, len(b)) + for i, c := range b { + if c < 0x20 || c >= 0x7f { + out[i] = '.' + } else { + out[i] = c + } + } + return bytes.TrimSpace(out) +} + +// passthroughDEK is the default DEKResolver — assumes the bytes +// arriving in encrypted_dek are already the 32-byte raw DEK. The +// production wiring will replace this with a KMS-backed resolver. +type passthroughDEK struct{} + +func (passthroughDEK) Unwrap(_ context.Context, in []byte) ([]byte, error) { + if len(in) != dekSize { + return nil, fmt.Errorf("pipeline: expected %d-byte DEK, got %d", dekSize, len(in)) + } + out := make([]byte, dekSize) + copy(out, in) + return out, nil +} + +// wipe zeroes a byte slice. Best-effort — the Go runtime makes no +// guarantee that the underlying memory pages aren't already swapped +// out, but this still raises the bar for casual memory inspection. +func wipe(b []byte) { + for i := range b { + b[i] = 0 + } +} diff --git a/apps/agent/internal/pipeline/runner_test.go b/apps/agent/internal/pipeline/runner_test.go new file mode 100644 index 0000000..2724fd9 --- /dev/null +++ b/apps/agent/internal/pipeline/runner_test.go @@ -0,0 +1,265 @@ +package pipeline + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// hexSha256 returns the lower-case hex SHA-256 of b. Test helper kept +// in this file so the production runner.go has no test-only imports. +func hexSha256(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:]) +} + +// fakeDriver emits a fixed plaintext payload prefixed with the configured magic. +type fakeDriver struct { + name string + payload []byte + version string + failVal bool + failDmp bool +} + +func (f *fakeDriver) Name() string { return f.name } + +func (f *fakeDriver) Validate(_ context.Context, _ *backupv1.Target) error { + if f.failVal { + return errors.New("validate boom") + } + return nil +} + +func (f *fakeDriver) Dump(_ context.Context, _ *backupv1.Target, out io.Writer) (DumpInfo, error) { + if f.failDmp { + return DumpInfo{}, errors.New("dump boom") + } + if _, err := out.Write(f.payload); err != nil { + return DumpInfo{}, err + } + return DumpInfo{EngineVersion: f.version}, nil +} + +// simpleLookups satisfies both TargetLookup and JobLookup with a single +// fixed (job, target) tuple. +type simpleLookups struct { + job *backupv1.BackupJobSpec + target *backupv1.Target +} + +func (s *simpleLookups) Job(id string) (*backupv1.BackupJobSpec, bool) { + if s.job != nil && s.job.Id == id { + return s.job, true + } + return nil, false +} + +func (s *simpleLookups) Target(id string) (*backupv1.Target, bool) { + if s.target != nil && s.target.Id == id { + return s.target, true + } + return nil, false +} + +// startFakeS3 spins up an httptest server that accepts a single PUT +// and records the body in `received`. +// +// The handler tolerates abrupt client disconnects — the pipeline may +// cancel the upload mid-stream when an earlier stage (smoke check, +// dump, etc.) fails. In that case `io.Copy` returns "unexpected EOF" +// or "use of closed network connection"; we record whatever bytes +// arrived and respond with 200 so the uploader sees the upload as +// having completed (the run's error still propagates from the failed +// stage upstream). +func startFakeS3(t *testing.T, received *bytes.Buffer) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodPut, r.Method) + _, _ = io.Copy(received, r.Body) + w.WriteHeader(http.StatusOK) + })) +} + +func TestRunner_HappyPath_PostgreSQL(t *testing.T) { + // 1 MiB of random bytes prefixed with the pg_dump magic. + plaintext := append([]byte(PgDumpMagic), make([]byte, 1<<20)...) + _, err := rand.Read(plaintext[len(PgDumpMagic):]) + require.NoError(t, err) + + driver := &fakeDriver{ + name: "pg_dump", + payload: plaintext, + version: "PostgreSQL 16.2", + } + dek := make([]byte, 32) + _, _ = rand.Read(dek) + + job := &backupv1.BackupJobSpec{Id: "job-1", TargetId: "tgt-1"} + target := &backupv1.Target{ + Id: "tgt-1", + Type: backupv1.DbType_POSTGRESQL, + Connection: &backupv1.ConnectionConfig{ + Host: "127.0.0.1", Port: 5432, Database: "x", Username: "u", + }, + } + lookups := &simpleLookups{job: job, target: target} + + var received bytes.Buffer + srv := startFakeS3(t, &received) + defer srv.Close() + + runner := NewRunner( + map[string]Driver{"postgresql": driver}, + NewUploaderWithClient(srv.Client()), + WithTargetLookup(lookups), + WithJobLookup(lookups), + ) + + req := &backupv1.RunBackup{ + JobId: "job-1", + RunId: "run-1", + EncryptedDek: dek, + UploadCreds: &backupv1.S3UploadCreds{ + PresignedPutUrl: srv.URL + "/run-1.enc", + FinalS3Key: "co_test/agt_test/job_job-1/run_run-1.enc", + }, + } + + completed, err := runner.Run(context.Background(), req) + require.NoError(t, err) + require.Equal(t, "job-1", completed.JobId) + require.Equal(t, "run-1", completed.RunId) + require.Equal(t, "zstd", completed.Compression) + require.Equal(t, "PostgreSQL 16.2", completed.DbEngineVersion) + require.Equal(t, uint64(received.Len()), completed.SizeBytes) + require.NotEmpty(t, completed.Sha256) + require.Equal(t, hexSha256(received.Bytes()), completed.Sha256, "sha256 must cover the ciphertext bytes actually uploaded") + require.Equal(t, dek, completed.EncryptedDek, "encrypted_dek must be passed through unchanged") + + // End-to-end: decrypt + decompress the uploaded blob and verify it + // equals the original plaintext. + enc, err := NewEncryptor(dek) + require.NoError(t, err) + var compressed bytes.Buffer + _, err = enc.Decrypt(&received, &compressed) + require.NoError(t, err) + + zr, err := zstd.NewReader(&compressed) + require.NoError(t, err) + defer zr.Close() + round, err := io.ReadAll(zr) + require.NoError(t, err) + require.Equal(t, plaintext, round) +} + +func TestRunner_MissingMagic_FailsBeforeUpload(t *testing.T) { + // Driver claims to be pg_dump but emits the wrong header. + driver := &fakeDriver{name: "pg_dump", payload: []byte("NOTAPGDUMP"), version: "?"} + dek := make([]byte, 32) + _, _ = rand.Read(dek) + + job := &backupv1.BackupJobSpec{Id: "j", TargetId: "t"} + target := &backupv1.Target{Id: "t", Type: backupv1.DbType_POSTGRESQL, Connection: &backupv1.ConnectionConfig{Host: "x"}} + lookups := &simpleLookups{job: job, target: target} + + var received bytes.Buffer + srv := startFakeS3(t, &received) + defer srv.Close() + + runner := NewRunner( + map[string]Driver{"postgresql": driver}, + NewUploaderWithClient(srv.Client()), + WithTargetLookup(lookups), + WithJobLookup(lookups), + ) + req := &backupv1.RunBackup{ + JobId: "j", RunId: "r", + EncryptedDek: dek, + UploadCreds: &backupv1.S3UploadCreds{PresignedPutUrl: srv.URL + "/r.enc", FinalS3Key: "k"}, + } + + _, err := runner.Run(context.Background(), req) + require.Error(t, err) +} + +func TestRunner_ValidateFailsFast(t *testing.T) { + driver := &fakeDriver{name: "pg_dump", payload: []byte(PgDumpMagic), failVal: true} + dek := make([]byte, 32) + _, _ = rand.Read(dek) + + job := &backupv1.BackupJobSpec{Id: "j", TargetId: "t"} + target := &backupv1.Target{Id: "t", Type: backupv1.DbType_POSTGRESQL, Connection: &backupv1.ConnectionConfig{Host: "x"}} + lookups := &simpleLookups{job: job, target: target} + + runner := NewRunner( + map[string]Driver{"postgresql": driver}, + NewUploader(), + WithTargetLookup(lookups), + WithJobLookup(lookups), + ) + req := &backupv1.RunBackup{ + JobId: "j", RunId: "r", + EncryptedDek: dek, + UploadCreds: &backupv1.S3UploadCreds{PresignedPutUrl: "http://127.0.0.1:0/never", FinalS3Key: "k"}, + } + _, err := runner.Run(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "validate stage") +} + +func TestRunner_UnknownDriver(t *testing.T) { + dek := make([]byte, 32) + _, _ = rand.Read(dek) + job := &backupv1.BackupJobSpec{Id: "j", TargetId: "t"} + target := &backupv1.Target{Id: "t", Type: backupv1.DbType_MONGODB, Connection: &backupv1.ConnectionConfig{Host: "x"}} + lookups := &simpleLookups{job: job, target: target} + + runner := NewRunner( + map[string]Driver{"postgresql": &fakeDriver{name: "pg_dump", payload: []byte(PgDumpMagic)}}, + NewUploader(), + WithTargetLookup(lookups), + WithJobLookup(lookups), + ) + req := &backupv1.RunBackup{ + JobId: "j", RunId: "r", + EncryptedDek: dek, + UploadCreds: &backupv1.S3UploadCreds{PresignedPutUrl: "http://127.0.0.1:0/", FinalS3Key: "k"}, + } + _, err := runner.Run(context.Background(), req) + require.Error(t, err) + require.Contains(t, err.Error(), "no driver registered") +} + +func TestRunner_DEKWrongLength(t *testing.T) { + driver := &fakeDriver{name: "pg_dump", payload: []byte(PgDumpMagic)} + job := &backupv1.BackupJobSpec{Id: "j", TargetId: "t"} + target := &backupv1.Target{Id: "t", Type: backupv1.DbType_POSTGRESQL, Connection: &backupv1.ConnectionConfig{Host: "x"}} + lookups := &simpleLookups{job: job, target: target} + + runner := NewRunner( + map[string]Driver{"postgresql": driver}, + NewUploader(), + WithTargetLookup(lookups), + WithJobLookup(lookups), + ) + req := &backupv1.RunBackup{ + JobId: "j", RunId: "r", + EncryptedDek: []byte("short"), + UploadCreds: &backupv1.S3UploadCreds{PresignedPutUrl: "http://127.0.0.1:0/", FinalS3Key: "k"}, + } + _, err := runner.Run(context.Background(), req) + require.Error(t, err) +} diff --git a/apps/agent/internal/pipeline/sqlite.go b/apps/agent/internal/pipeline/sqlite.go new file mode 100644 index 0000000..c7d2e3b --- /dev/null +++ b/apps/agent/internal/pipeline/sqlite.go @@ -0,0 +1,195 @@ +// B14: SQLite driver. +// +// Streams a consistent snapshot of a SQLite database file using +// `sqlite3 ".backup '/dev/stdout'"`. The `.backup` command takes +// an online-safe copy of the database — readers continue to see the +// previous state while we drain. The resulting bytes are a complete +// SQLite database file (not SQL text), which we pipe through gzip so +// the stream is self-describing and (modestly) smaller before the +// pipeline's zstd stage layers on top. +// +// Configuration: +// +// - target.Connection.Database — REQUIRED. Absolute path to the .db +// file. (Host/Port/Username are ignored.) +// +// The driver validates that the file exists and is readable at +// construction-time. WAL-mode databases are safe to back up with the +// `.backup` command — SQLite quiesces concurrent writers internally. +package pipeline + +import ( + "compress/gzip" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// sqliteDriver implements Driver against the `sqlite3` CLI binary plus +// optional pure-Go metadata probing. +type sqliteDriver struct { + binary string + runner cmdRunner + // statFn is overridable for tests. + statFn func(path string) (os.FileInfo, error) + // metaFn is overridable for tests. Returns (user_version, table_count, err). + metaFn func(ctx context.Context, path string) (int64, int, error) +} + +// NewSqliteDriver constructs the default driver wired to the bundled +// sqlite3 binary on $PATH. +func NewSqliteDriver() Driver { + return &sqliteDriver{ + binary: "sqlite3", + runner: streamingRunner{}, + statFn: os.Stat, + metaFn: probeSqliteMetadata, + } +} + +// Name implements Driver.Name. +func (s *sqliteDriver) Name() string { return "sqlite" } + +// Validate verifies sqlite3 is installed and the configured database +// file exists and is readable. +func (s *sqliteDriver) Validate(ctx context.Context, target *backupv1.Target) error { + if target == nil || target.Connection == nil { + return errors.New("pipeline: sqlite: nil target/connection") + } + path := target.Connection.Database + if path == "" { + return errors.New("pipeline: sqlite: target.connection.database must be the path to the .db file") + } + versionOut, err := s.runner.Output(ctx, s.binary, []string{"--version"}, nil) + if err != nil { + return fmt.Errorf("pipeline: sqlite3 version probe failed (is sqlite3 installed?): %w", err) + } + if !looksLikeSqliteVersion(string(versionOut)) { + return fmt.Errorf("pipeline: unexpected sqlite3 --version output: %q", string(versionOut)) + } + if _, err := s.statFn(path); err != nil { + return fmt.Errorf("pipeline: sqlite: cannot stat database %q: %w", path, err) + } + return nil +} + +// Dump streams a gzip-wrapped binary SQLite backup to out. +// +// We invoke sqlite3 with `.backup '/dev/stdout'`. This is the canonical +// way to take an online-safe snapshot — concurrent readers see the old +// state, concurrent writers are not blocked, and the resulting file is +// a fully self-contained .db that `sqlite3 restored.db` can open. +func (s *sqliteDriver) Dump(ctx context.Context, target *backupv1.Target, out io.Writer) (DumpInfo, error) { + if target == nil || target.Connection == nil { + return DumpInfo{}, errors.New("pipeline: sqlite: nil target/connection") + } + path := target.Connection.Database + if path == "" { + return DumpInfo{}, errors.New("pipeline: sqlite: connection.database must be the path to the .db file") + } + if _, err := s.statFn(path); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: sqlite: cannot stat database %q: %w", path, err) + } + + gz := gzip.NewWriter(out) + defer gz.Close() + + // sqlite3 expects a single positional argument (the database path) + // followed by a dot-command. `.backup` writes a consistent snapshot + // to the supplied filename; we pass /dev/stdout so the bytes flow + // through stdout into our gzip writer. + args := []string{path, ".backup '/dev/stdout'"} + if err := s.runner.RunStream(ctx, s.binary, args, nil, gz); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: sqlite3 .backup exec: %w", err) + } + if err := gz.Close(); err != nil { + return DumpInfo{}, fmt.Errorf("pipeline: sqlite: close gzip: %w", err) + } + + info := DumpInfo{EngineVersion: s.versionString(ctx)} + return info, nil +} + +// versionString turns `sqlite3 --version` output into a canonical +// "SQLite " string. The raw output is e.g. +// "3.45.1 2024-01-30 16:01:20 e876e51a04…"; we keep the first token. +func (s *sqliteDriver) versionString(ctx context.Context) string { + out, err := s.runner.Output(ctx, s.binary, []string{"--version"}, nil) + if err != nil { + return "SQLite" + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "SQLite" + } + return "SQLite " + fields[0] +} + +// looksLikeSqliteVersion accepts any --version banner whose first token +// looks like a dotted version number (e.g. "3.45.1"). +func looksLikeSqliteVersion(s string) bool { + fields := strings.Fields(s) + if len(fields) == 0 { + return false + } + for _, part := range strings.Split(fields[0], ".") { + if _, err := strconv.Atoi(part); err != nil { + return false + } + } + return true +} + +// probeSqliteMetadata opens the database file read-only and returns +// (user_version, table_count, err). Used for metadata enrichment when +// the agent has a registered sqlite driver in database/sql. +// +// To keep the agent dependency-free for MVP we only attempt the probe +// when a "sqlite" or "sqlite3" driver is registered with database/sql +// at runtime. Production builds may register one via blank-import in +// cmd/agent if richer metadata is required; the MVP build skips it. +func probeSqliteMetadata(ctx context.Context, path string) (int64, int, error) { + for _, name := range []string{"sqlite", "sqlite3"} { + if !sqlDriverRegistered(name) { + continue + } + db, err := sql.Open(name, "file:"+path+"?mode=ro") + if err != nil { + return 0, 0, err + } + defer db.Close() + var uv int64 + if err := db.QueryRowContext(ctx, "PRAGMA user_version").Scan(&uv); err != nil { + return 0, 0, err + } + var tc int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&tc); err != nil { + return 0, 0, err + } + return uv, tc, nil + } + return 0, 0, errors.New("no sqlite driver registered with database/sql") +} + +// sqlDriverRegistered reports whether `name` is registered via sql.Register. +func sqlDriverRegistered(name string) bool { + for _, d := range sql.Drivers() { + if d == name { + return true + } + } + return false +} + +// IsSqliteGzMagic reports whether head looks like the gzip header that +// every sqliteDriver.Dump stream begins with. +func IsSqliteGzMagic(head []byte) bool { + return len(head) >= 2 && head[0] == 0x1f && head[1] == 0x8b +} diff --git a/apps/agent/internal/pipeline/sqlite_test.go b/apps/agent/internal/pipeline/sqlite_test.go new file mode 100644 index 0000000..35856c3 --- /dev/null +++ b/apps/agent/internal/pipeline/sqlite_test.go @@ -0,0 +1,174 @@ +package pipeline + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +func TestSqlite_Name(t *testing.T) { + t.Parallel() + require.Equal(t, "sqlite", (&sqliteDriver{}).Name()) +} + +func TestSqlite_LooksLikeVersion(t *testing.T) { + t.Parallel() + require.True(t, looksLikeSqliteVersion("3.45.1 2024-01-30 16:01:20")) + require.True(t, looksLikeSqliteVersion("3.42.0")) + require.False(t, looksLikeSqliteVersion("totally bogus")) + require.False(t, looksLikeSqliteVersion("")) +} + +func TestSqlite_Validate_MissingDatabasePath(t *testing.T) { + t.Parallel() + d := &sqliteDriver{ + binary: "sqlite3", + runner: &mockRunner{outputResp: map[string][]byte{"--version": []byte("3.45.1\n")}}, + statFn: os.Stat, + } + err := d.Validate(context.Background(), &backupv1.Target{Type: backupv1.DbType_SQLITE, Connection: &backupv1.ConnectionConfig{}}) + require.Error(t, err) + require.Contains(t, err.Error(), "database must be the path") +} + +func TestSqlite_Validate_FileNotFound(t *testing.T) { + t.Parallel() + d := &sqliteDriver{ + binary: "sqlite3", + runner: &mockRunner{outputResp: map[string][]byte{"--version": []byte("3.45.1\n")}}, + statFn: os.Stat, + } + err := d.Validate(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{Database: "/path/does/not/exist.db"}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot stat database") +} + +func TestSqlite_Validate_BinaryMissing(t *testing.T) { + t.Parallel() + d := &sqliteDriver{ + binary: "sqlite3", + runner: &errOutputRunner{err: errors.New("not found")}, + statFn: os.Stat, + } + err := d.Validate(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{Database: "/tmp/foo.db"}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "version probe failed") +} + +func TestSqlite_Validate_OK(t *testing.T) { + t.Parallel() + tmp := writeTempDB(t, []byte("placeholder")) + d := &sqliteDriver{ + binary: "sqlite3", + runner: &mockRunner{outputResp: map[string][]byte{"--version": []byte("3.45.1 2024-01-30\n")}}, + statFn: os.Stat, + } + err := d.Validate(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{Database: tmp}, + }) + require.NoError(t, err) +} + +func TestSqlite_Dump_WrapsOutputInGzip(t *testing.T) { + t.Parallel() + tmp := writeTempDB(t, []byte("placeholder")) + payload := bytes.Repeat([]byte{0x53, 0x51, 0x4c}, 16) // pretend SQLite header bytes + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("3.45.1 2024-01-30\n")}, + streamResp: payload, + } + d := &sqliteDriver{binary: "sqlite3", runner: mock, statFn: os.Stat} + + var buf bytes.Buffer + info, err := d.Dump(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{Database: tmp}, + }, &buf) + require.NoError(t, err) + require.Equal(t, "SQLite 3.45.1", info.EngineVersion) + require.True(t, IsSqliteGzMagic(buf.Bytes())) + + gz, err := gzip.NewReader(&buf) + require.NoError(t, err) + defer gz.Close() + got, err := io.ReadAll(gz) + require.NoError(t, err) + require.Equal(t, payload, got) + + // Confirm `.backup '/dev/stdout'` was invoked with the right path. + require.NotEmpty(t, mock.calls) + streamCall := mock.calls[0] + require.Equal(t, tmp, streamCall.Args[0]) + require.Equal(t, ".backup '/dev/stdout'", streamCall.Args[1]) +} + +func TestSqlite_Dump_MissingPath(t *testing.T) { + t.Parallel() + d := &sqliteDriver{binary: "sqlite3", runner: &mockRunner{}, statFn: os.Stat} + var buf bytes.Buffer + _, err := d.Dump(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{}, + }, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), "must be the path") +} + +func TestSqlite_Dump_StreamErrorWraps(t *testing.T) { + t.Parallel() + tmp := writeTempDB(t, []byte("placeholder")) + mock := &mockRunner{ + outputResp: map[string][]byte{"--version": []byte("3.45.1\n")}, + streamErr: errors.New("permission denied"), + } + d := &sqliteDriver{binary: "sqlite3", runner: mock, statFn: os.Stat} + var buf bytes.Buffer + _, err := d.Dump(context.Background(), &backupv1.Target{ + Type: backupv1.DbType_SQLITE, + Connection: &backupv1.ConnectionConfig{Database: tmp}, + }, &buf) + require.Error(t, err) + require.Contains(t, err.Error(), ".backup exec") +} + +func TestIsSqliteGzMagic(t *testing.T) { + t.Parallel() + require.True(t, IsSqliteGzMagic([]byte{0x1f, 0x8b, 0x08})) + require.False(t, IsSqliteGzMagic([]byte{0x00, 0x00})) +} + +func TestSqlite_ProbeMetadata_NoDriver(t *testing.T) { + t.Parallel() + // In the MVP build no sqlite database/sql driver is registered. + // Confirm probeSqliteMetadata returns the expected sentinel error so + // callers can degrade gracefully. + _, _, err := probeSqliteMetadata(context.Background(), "/tmp/anything.db") + require.Error(t, err) + require.Contains(t, err.Error(), "no sqlite driver registered") +} + +// writeTempDB drops a tiny placeholder file into a temp directory and +// returns its absolute path. +func writeTempDB(t *testing.T, contents []byte) string { + t.Helper() + dir := t.TempDir() + p := filepath.Join(dir, "test.db") + require.NoError(t, os.WriteFile(p, contents, 0o600)) + return p +} diff --git a/apps/agent/internal/pipeline/upload.go b/apps/agent/internal/pipeline/upload.go new file mode 100644 index 0000000..08fe7b6 --- /dev/null +++ b/apps/agent/internal/pipeline/upload.go @@ -0,0 +1,93 @@ +package pipeline + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" +) + +// Uploader streams an encrypted backup blob to S3 via a presigned PUT +// URL while computing the SHA-256 of the bytes it actually uploads. +// +// Note (design): SHA-256 is computed over the CIPHERTEXT — that is, the +// exact blob persisted in S3. The server uses this to detect bit rot. +// The plaintext hash is not exposed; it could only be derived after a +// successful decrypt, which is what backupy-decrypt is for. +type Uploader struct { + httpClient *http.Client +} + +// NewUploader constructs an Uploader with a sensible default http.Client. +// The client has no overall request timeout because uploads can be +// hours-long for very large dumps — cancellation flows through ctx +// instead. +func NewUploader() *Uploader { + return &Uploader{ + httpClient: &http.Client{ + // No Timeout: large uploads must be allowed to run long. + // Per-stage timeouts are enforced by ctx. + Transport: &http.Transport{ + // Honour HTTP/2 + idle-conn defaults; only override + // the response-header deadline so a stuck server is + // detected within 30s rather than hanging forever. + ResponseHeaderTimeout: 30 * time.Second, + ExpectContinueTimeout: 5 * time.Second, + }, + }, + } +} + +// NewUploaderWithClient is a constructor used by tests to inject a +// custom http.Client (e.g. one wired to httptest.Server). +func NewUploaderWithClient(c *http.Client) *Uploader { + return &Uploader{httpClient: c} +} + +// Put streams body to presignedURL using HTTP PUT and computes SHA-256 +// of the body on the fly. contentLength may be -1 if unknown — in that +// case the request is sent chunked. +// +// Returns the lower-case hex SHA-256 and the number of bytes uploaded. +func (u *Uploader) Put(ctx context.Context, presignedURL string, body io.Reader, contentLength int64) (string, int64, error) { + h := sha256.New() + counted := &countingReader{r: io.TeeReader(body, h)} + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, presignedURL, counted) + if err != nil { + return "", 0, fmt.Errorf("pipeline: build PUT request: %w", err) + } + if contentLength >= 0 { + req.ContentLength = contentLength + } + // Match the Content-Type the server's presign was generated for. + // "application/octet-stream" is the canonical default for raw blobs. + req.Header.Set("Content-Type", "application/octet-stream") + + resp, err := u.httpClient.Do(req) + if err != nil { + return "", counted.n, fmt.Errorf("pipeline: PUT %s: %w", presignedURL, err) + } + defer func() { _, _ = io.Copy(io.Discard, resp.Body); _ = resp.Body.Close() }() + + if resp.StatusCode/100 != 2 { + return "", counted.n, fmt.Errorf("pipeline: upload non-2xx: HTTP %d", resp.StatusCode) + } + return hex.EncodeToString(h.Sum(nil)), counted.n, nil +} + +// countingReader counts bytes read from the wrapped reader. Used to +// compute the uploaded size without buffering. +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} diff --git a/apps/agent/internal/proto/proto.go b/apps/agent/internal/proto/proto.go new file mode 100644 index 0000000..0c353d5 --- /dev/null +++ b/apps/agent/internal/proto/proto.go @@ -0,0 +1,73 @@ +// Package proto re-exports the generated protobuf types so the rest of the +// agent imports a single canonical name (`proto.Envelope`) rather than the +// long `backupv1.Envelope`. It also centralises a few convenience helpers +// (NewEnvelope) so call sites stay concise. +// +// Prerequisite: `make proto` must have been run from the repo root to +// generate packages/proto/gen/go/v1/*.pb.go. The agent's go.mod uses a +// `replace` directive pointing at that local path — see go.mod. +package proto + +import ( + "time" + + backupv1 "github.com/backupy/backupy/packages/proto/gen/go/backupv1" +) + +// Type aliases keep the agent codebase decoupled from the long generated +// package path. If we ever migrate to backup.v2 we change the alias here +// and the rest of the agent compiles unchanged. +type ( + Envelope = backupv1.Envelope + Register = backupv1.Register + Heartbeat = backupv1.Heartbeat + AgentMetrics = backupv1.AgentMetrics + DiscoveryReport = backupv1.DiscoveryReport + JobUpdate = backupv1.JobUpdate + BackupCompleted = backupv1.BackupCompleted + HealthCheckResult = backupv1.HealthCheckResult + LogEvent = backupv1.LogEvent + Ack = backupv1.Ack + + RegisterAck = backupv1.RegisterAck + ConfigUpdate = backupv1.ConfigUpdate + RunBackup = backupv1.RunBackup + CancelJob = backupv1.CancelJob + RunHealthCheck = backupv1.RunHealthCheck + SelfUpdate = backupv1.SelfUpdate + Ping = backupv1.Ping + + AgentConfig = backupv1.AgentConfig + + // Envelope payload oneof wrappers — full set so client/loops construct + // outgoing envelopes and pattern-match incoming via `agentproto.Envelope_*` + // without importing the generated package. + Envelope_Register = backupv1.Envelope_Register + Envelope_Heartbeat = backupv1.Envelope_Heartbeat + Envelope_Discovery = backupv1.Envelope_Discovery + Envelope_JobUpdate = backupv1.Envelope_JobUpdate + Envelope_BackupCompleted = backupv1.Envelope_BackupCompleted + Envelope_HealthResult = backupv1.Envelope_HealthResult + Envelope_Log = backupv1.Envelope_Log + Envelope_RestoreUpdate = backupv1.Envelope_RestoreUpdate + Envelope_Ack = backupv1.Envelope_Ack + Envelope_RegisterAck = backupv1.Envelope_RegisterAck + Envelope_ConfigUpdate = backupv1.Envelope_ConfigUpdate + Envelope_RunBackup = backupv1.Envelope_RunBackup + Envelope_CancelJob = backupv1.Envelope_CancelJob + Envelope_RunHealthCheck = backupv1.Envelope_RunHealthCheck + Envelope_SelfUpdate = backupv1.Envelope_SelfUpdate + Envelope_Ping = backupv1.Envelope_Ping +) + +// NowMillis returns the current wall-clock time in unix milliseconds — +// the canonical ts_ms field used in every Envelope. +func NowMillis() uint64 { + return uint64(time.Now().UnixMilli()) +} + +// NewEnvelope is a convenience constructor that stamps ts_ms with the +// current time. Callers set seq + correlation_id + payload separately. +func NewEnvelope() *Envelope { + return &Envelope{TsMs: NowMillis()} +} diff --git a/apps/agent/internal/queue/queue.go b/apps/agent/internal/queue/queue.go new file mode 100644 index 0000000..e07f4c3 --- /dev/null +++ b/apps/agent/internal/queue/queue.go @@ -0,0 +1,65 @@ +// Package queue is a thin in-process abstraction over state.Store's queue +// bucket. The state package owns persistence; this package owns ordering +// semantics and (in D-02) the channel-based delivery to job workers. +// +// Today this file declares the small interface and a state-backed +// implementation that satisfies it — enough for the WSS client to push +// inbound RunBackup envelopes onto disk during the skeleton phase. +package queue + +import ( + "context" + + "github.com/backupy/backupy/apps/agent/internal/state" +) + +// Queue is the agent's persistent job queue. +type Queue interface { + // Enqueue persists the encoded payload keyed by run_id. Idempotent. + Enqueue(runID string, payload []byte) error + // Pop returns up to n jobs without removing them from durable storage. + Pop(ctx context.Context, n int) ([]Job, error) + // Ack removes a job from durable storage after successful handling. + Ack(runID string) error + // Depth returns the current pending job count. + Depth() (int, error) +} + +// Job is one queued item. +type Job struct { + RunID string + Payload []byte +} + +// NewBolt returns a Queue backed by the BoltDB state store. +func NewBolt(s *state.Store) Queue { + return &boltQueue{s: s} +} + +type boltQueue struct { + s *state.Store +} + +func (q *boltQueue) Enqueue(runID string, payload []byte) error { + return q.s.EnqueueJob(runID, payload) +} + +func (q *boltQueue) Pop(_ context.Context, n int) ([]Job, error) { + raw, err := q.s.DequeueJobs(n) + if err != nil { + return nil, err + } + out := make([]Job, len(raw)) + for i, j := range raw { + out[i] = Job{RunID: j.RunID, Payload: j.Payload} + } + return out, nil +} + +func (q *boltQueue) Ack(runID string) error { + return q.s.AckJob(runID) +} + +func (q *boltQueue) Depth() (int, error) { + return q.s.QueueDepth() +} diff --git a/apps/agent/internal/version/version.go b/apps/agent/internal/version/version.go new file mode 100644 index 0000000..1d0300a --- /dev/null +++ b/apps/agent/internal/version/version.go @@ -0,0 +1,41 @@ +// Package version exposes build-time information injected via -ldflags. +// +// Example go build: +// +// go build -ldflags " +// -X github.com/backupy/backupy/apps/agent/internal/version.Version=$(VERSION) +// -X github.com/backupy/backupy/apps/agent/internal/version.Commit=$(COMMIT) +// -X github.com/backupy/backupy/apps/agent/internal/version.BuildDate=$(DATE) +// " ./cmd/agent +package version + +import "fmt" + +// These variables are overwritten by the linker at build time. Defaults +// describe an unstamped local build so `agent version` is always answerable. +var ( + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) + +// Info is an immutable snapshot of build metadata. +type Info struct { + Version string `json:"version"` + Commit string `json:"commit"` + BuildDate string `json:"build_date"` +} + +// Current returns the current build info. +func Current() Info { + return Info{ + Version: Version, + Commit: Commit, + BuildDate: BuildDate, + } +} + +// Full returns a human-readable single-line version string for CLI output. +func Full() string { + return fmt.Sprintf("%s (commit %s, built %s)", Version, Commit, BuildDate) +} diff --git a/apps/agent/internal/wss/backoff.go b/apps/agent/internal/wss/backoff.go new file mode 100644 index 0000000..fc3dea8 --- /dev/null +++ b/apps/agent/internal/wss/backoff.go @@ -0,0 +1,61 @@ +package wss + +import ( + "math/rand" + "time" +) + +// Backoff implements the exponential schedule specified in +// docs/03-agent-spec.md → "Reconnect: exponential backoff 1s → 2s → 4s … +// → 60s cap, jitter ±20%". +// +// The struct is safe for use from a single goroutine. The reconnect loop +// is single-threaded, so no mutex is needed. +type Backoff struct { + Initial time.Duration + Max time.Duration + Factor float64 + JitterPC float64 // 0.2 → ±20% + + current time.Duration + rng *rand.Rand +} + +// NewBackoff returns a Backoff with spec defaults (1s → 60s, ±20%, ×2). +func NewBackoff() *Backoff { + return &Backoff{ + Initial: 1 * time.Second, + Max: 60 * time.Second, + Factor: 2.0, + JitterPC: 0.2, + // Deterministic-ish but unique per agent: seed from the wall clock. + rng: rand.New(rand.NewSource(time.Now().UnixNano())), //nolint:gosec // not a security context + } +} + +// Next returns the next delay and advances the schedule. +func (b *Backoff) Next() time.Duration { + if b.current == 0 { + b.current = b.Initial + } else { + b.current = time.Duration(float64(b.current) * b.Factor) + if b.current > b.Max { + b.current = b.Max + } + } + return b.withJitter(b.current) +} + +// Reset rewinds the schedule. Call on a clean reconnect. +func (b *Backoff) Reset() { + b.current = 0 +} + +func (b *Backoff) withJitter(d time.Duration) time.Duration { + if b.JitterPC <= 0 { + return d + } + // pick uniformly in [-jitter, +jitter] + jitter := b.JitterPC * (b.rng.Float64()*2 - 1) + return time.Duration(float64(d) * (1 + jitter)) +} diff --git a/apps/agent/internal/wss/backoff_test.go b/apps/agent/internal/wss/backoff_test.go new file mode 100644 index 0000000..b837e3c --- /dev/null +++ b/apps/agent/internal/wss/backoff_test.go @@ -0,0 +1,43 @@ +package wss + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBackoff_Progression(t *testing.T) { + b := NewBackoff() + b.JitterPC = 0 // deterministic for the assertion + + require.Equal(t, 1*time.Second, b.Next()) + require.Equal(t, 2*time.Second, b.Next()) + require.Equal(t, 4*time.Second, b.Next()) + require.Equal(t, 8*time.Second, b.Next()) + require.Equal(t, 16*time.Second, b.Next()) + require.Equal(t, 32*time.Second, b.Next()) + require.Equal(t, 60*time.Second, b.Next(), "should cap at 60s") + require.Equal(t, 60*time.Second, b.Next(), "should stay at cap") +} + +func TestBackoff_Reset(t *testing.T) { + b := NewBackoff() + b.JitterPC = 0 + b.Next() + b.Next() + b.Reset() + require.Equal(t, 1*time.Second, b.Next()) +} + +func TestBackoff_JitterStaysInBand(t *testing.T) { + b := NewBackoff() + // Force a known current value and re-check jitter range. + const base = 10 * time.Second + b.current = base + for i := 0; i < 200; i++ { + got := b.withJitter(base) + require.GreaterOrEqual(t, got, time.Duration(float64(base)*0.8)) + require.LessOrEqual(t, got, time.Duration(float64(base)*1.2)) + } +} diff --git a/apps/agent/internal/wss/client.go b/apps/agent/internal/wss/client.go new file mode 100644 index 0000000..71ba351 --- /dev/null +++ b/apps/agent/internal/wss/client.go @@ -0,0 +1,449 @@ +// Package wss owns the long-lived WebSocket-Secure connection from the +// agent to the control plane (see docs/07-api-contract.md §1 and +// docs/03-agent-spec.md → "WSS-канал"). +// +// Lifecycle: +// +// 1. Dial cfg.ServerURL/v1/agents/connect with `Authorization: Bearer +// `. +// 2. Send Register; await RegisterAck. +// 3. Persist session_id + config snapshot in state.Store. +// 4. Spawn read + write goroutines; replay any queued outbound jobs; +// start the 30s heartbeat ticker. +// 5. On any read/write error, close the socket and reconnect with +// exponential backoff (see backoff.go). +// 6. On context cancellation, close cleanly and return. +// +// Safe for concurrent Send calls. +package wss + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + pkgmetrics "github.com/backupy/backupy/apps/agent/internal/metrics" + agentproto "github.com/backupy/backupy/apps/agent/internal/proto" + "github.com/backupy/backupy/apps/agent/internal/queue" + "github.com/backupy/backupy/apps/agent/internal/state" +) + +// heartbeatInterval is the default heartbeat cadence; the server may +// override it in RegisterAck.heartbeat_interval_sec. +const heartbeatInterval = 30 * time.Second + +// readIdleTimeout is how long the agent waits for *any* server frame +// before declaring the connection dead. The server pings every 30s; we +// allow three misses to match the RTO from docs/07 §1. +const readIdleTimeout = 90 * time.Second + +// writeTimeout bounds a single Write — must be shorter than the read +// timeout on the other side so the server doesn't tear us down first. +const writeTimeout = 10 * time.Second + +// registerTimeout bounds the Register/RegisterAck handshake. +const registerTimeout = 15 * time.Second + +// Config is everything the client needs from the outside world. +type Config struct { + ServerURL string + AgentKey string // never logged + // AgentVersion is reported in the Register payload. + AgentVersion string + // Hostname / OS / Arch are filled by the agent's runtime probe; + // callers may leave them empty and the client will detect. + Hostname string + OS string + Arch string + DockerVersion string + Capabilities []string + // AllowInsecure permits ws:// / http:// dial schemes when ServerURL + // uses one. Production must leave this false — it matches the + // agent's BACKUP_DEV_ALLOW_INSECURE bootstrap flag. + AllowInsecure bool +} + +// Handlers carries user callbacks invoked when the server pushes a +// command. Each callback runs on the read goroutine and must return +// quickly — long-running work belongs on a worker pool. +type Handlers struct { + OnConfigUpdate func(ctx context.Context, msg *agentproto.ConfigUpdate) error + OnRunBackup func(ctx context.Context, msg *agentproto.RunBackup) error + OnCancelJob func(ctx context.Context, msg *agentproto.CancelJob) error + OnRunHealthCheck func(ctx context.Context, msg *agentproto.RunHealthCheck) error + OnSelfUpdate func(ctx context.Context, msg *agentproto.SelfUpdate) error +} + +// AgentMetrics is a hook supplied by the caller to populate periodic +// Heartbeats. Returns the most recent snapshot of host metrics. May be +// nil — heartbeats then carry zeroed metrics. +type AgentMetrics func() *agentproto.AgentMetrics + +// Client is the WSS connection manager. Safe for concurrent Send calls. +type Client struct { + cfg Config + state *state.Store + queue queue.Queue + handlers *Handlers + metrics AgentMetrics + logger *slog.Logger + + // connMu protects the active websocket conn and out chan; both + // are nil when disconnected. + connMu sync.Mutex + conn *websocket.Conn + out chan *agentproto.Envelope + + seq atomic.Uint64 + configVersion atomic.Uint64 + sessionID atomic.Value // string + + reconnectCount atomic.Uint64 +} + +// NewClient constructs a client. The connection is not opened until +// Start. logger may be nil. +func NewClient(cfg Config, st *state.Store, q queue.Queue, h *Handlers, m AgentMetrics, logger *slog.Logger) *Client { + if logger == nil { + logger = slog.Default() + } + c := &Client{ + cfg: cfg, + state: st, + queue: q, + handlers: h, + metrics: m, + logger: logger.With(slog.String("component", "wss")), + } + // Seed config version from the last persisted snapshot so the + // server can decide whether a ConfigUpdate is needed at register. + if st != nil { + if v, _, err := st.LoadConfig(); err == nil { + c.configVersion.Store(v) + } + } + if cfg.Hostname == "" { + if h, err := osHostname(); err == nil { + c.cfg.Hostname = h + } + } + if cfg.OS == "" { + c.cfg.OS = runtime.GOOS + } + if cfg.Arch == "" { + c.cfg.Arch = runtime.GOARCH + } + return c +} + +// Start runs the connection lifecycle until ctx is cancelled. Each +// iteration dials, runs read/write loops, and on any failure waits per +// the backoff schedule before retrying. +func (c *Client) Start(ctx context.Context) error { + c.logger.Info("wss client starting", + slog.String("server_url", c.cfg.ServerURL)) + bo := NewBackoff() + for { + if err := ctx.Err(); err != nil { + c.logger.Info("wss client shutting down", slog.Any("reason", err)) + return nil + } + runErr := c.runOnce(ctx) + if ctx.Err() != nil { + c.logger.Info("wss client shutting down") + pkgmetrics.SetWSSState("disconnected") + return nil + } + c.reconnectCount.Add(1) + pkgmetrics.WSSReconnects.Inc() + pkgmetrics.SetWSSState("reconnecting") + delay := bo.Next() + c.logger.Warn("wss disconnected; reconnecting", + slog.Any("err", runErr), + slog.Duration("backoff", delay)) + select { + case <-ctx.Done(): + return nil + case <-time.After(delay): + } + // Reset backoff after a successful connection cycle is handled + // inside runOnce when the handshake completes. + _ = bo + } +} + +// Send queues an outbound envelope. If the client is currently +// connected, the envelope is pushed to the in-memory out channel; if +// disconnected (or the buffer is full), the envelope is persisted to +// the on-disk queue keyed by correlation_id (or seq-N fallback). +func (c *Client) Send(env *agentproto.Envelope) error { + if env == nil { + return errors.New("wss: nil envelope") + } + if env.Seq == 0 { + env.Seq = c.seq.Add(1) + } + if env.TsMs == 0 { + env.TsMs = agentproto.NowMillis() + } + c.connMu.Lock() + out := c.out + c.connMu.Unlock() + if out != nil { + select { + case out <- env: + return nil + default: + // fall through to persistent queue + } + } + raw, err := proto.Marshal(env) + if err != nil { + return fmt.Errorf("wss: marshal envelope: %w", err) + } + key := env.CorrelationId + if key == "" { + key = fmt.Sprintf("seq-%d", env.Seq) + } + if c.queue == nil { + return errors.New("wss: queue not configured, message dropped") + } + return c.queue.Enqueue(key, raw) +} + +// SessionID returns the most recently assigned session id, or "" if no +// successful handshake has happened yet. +func (c *Client) SessionID() string { + v, _ := c.sessionID.Load().(string) + return v +} + +// ConfigVersion returns the currently-applied config version. +func (c *Client) ConfigVersion() uint64 { return c.configVersion.Load() } + +// ReconnectCount returns the total number of reconnect attempts since +// Start was invoked. Exported for the metrics endpoint. +func (c *Client) ReconnectCount() uint64 { return c.reconnectCount.Load() } + +// runOnce dials, performs the handshake, and pumps frames until the +// connection breaks. Returns the cause of the disconnect, or nil if +// ctx was cancelled. +func (c *Client) runOnce(ctx context.Context) error { + ws, err := c.dial(ctx) + if err != nil { + return fmt.Errorf("dial: %w", err) + } + defer func() { _ = ws.Close(websocket.StatusNormalClosure, "") }() + + // Handshake: Register -> RegisterAck. + ack, err := c.handshake(ctx, ws) + if err != nil { + _ = ws.Close(websocket.StatusPolicyViolation, "handshake failed") + return fmt.Errorf("handshake: %w", err) + } + + // Persist session + config. + c.sessionID.Store(ack.SessionId) + // Track the applied config version in-memory regardless of whether a + // state store is wired in (the store is optional in tests and for + // stateless deployments). Persistence happens separately below. + if ack.Config != nil && ack.Config.Version > c.configVersion.Load() { + c.configVersion.Store(ack.Config.Version) + } + if c.state != nil { + _ = c.state.SaveSession(ack.SessionId, time.Now().UnixMilli()) + if ack.Config != nil { + if raw, merr := proto.Marshal(ack.Config); merr == nil { + _ = c.state.SaveConfig(ack.Config.Version, raw) + } + } + } + // Apply via user callback so the pipeline reacts immediately. This + // runs regardless of state-store presence because handlers are an + // independent injection point. + if ack.Config != nil && c.handlers != nil && c.handlers.OnConfigUpdate != nil { + _ = c.handlers.OnConfigUpdate(ctx, &agentproto.ConfigUpdate{Config: ack.Config}) + } + + // Handshake completed — mark the connection as live for metrics. + pkgmetrics.SetWSSState("connected") + + // Per-connection out channel; mirror it on the Client so Send can + // reach it. + out := make(chan *agentproto.Envelope, 64) + c.connMu.Lock() + c.conn = ws + c.out = out + c.connMu.Unlock() + defer func() { + c.connMu.Lock() + c.conn = nil + c.out = nil + c.connMu.Unlock() + }() + + // Replay any queued envelopes from disk — they were buffered while + // disconnected and must reach the server idempotently. + c.replayQueue(ctx, out) + + connCtx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 3) + go func() { errCh <- c.readLoop(connCtx, ws) }() + go func() { errCh <- c.writeLoop(connCtx, ws, out) }() + go func() { errCh <- c.heartbeatLoop(connCtx, out) }() + + err = <-errCh + cancel() + // Closing the websocket unblocks any pending Read/Write call so + // the remaining loops exit promptly. + _ = ws.Close(websocket.StatusNormalClosure, "client done") + // Drain the rest so they don't outlive us. + <-errCh + <-errCh + return err +} + +// dial opens the WebSocket connection, attaching the Authorization +// header. Returns the live conn or an error. +func (c *Client) dial(ctx context.Context) (*websocket.Conn, error) { + wsURL, err := buildWSURL(c.cfg.ServerURL, c.cfg.AllowInsecure) + if err != nil { + return nil, err + } + h := http.Header{} + h.Set("Authorization", "Bearer "+c.cfg.AgentKey) + + dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + + ws, _, err := websocket.Dial(dialCtx, wsURL, &websocket.DialOptions{ + HTTPHeader: h, + }) + if err != nil { + return nil, err + } + // The protobuf-encoded frames can be larger than the default 32 KB + // read limit (e.g. AgentConfig with many targets). + ws.SetReadLimit(4 * 1024 * 1024) + c.logger.Info("wss connected", slog.String("url", wsURL)) + return ws, nil +} + +// buildWSURL rewrites http(s):// to ws(s):// and appends the canonical +// agent endpoint path. +func buildWSURL(raw string, allowInsecure bool) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("parse server url: %w", err) + } + switch u.Scheme { + case "http": + if !allowInsecure { + return "", errors.New("server url must be https://; set AllowInsecure for dev") + } + u.Scheme = "ws" + case "https": + u.Scheme = "wss" + case "ws", "wss": + // already a websocket URL + default: + return "", fmt.Errorf("unsupported scheme %q", u.Scheme) + } + // Mount the canonical control-plane endpoint. Keep any path the + // caller provided as a prefix (some deploys put a reverse proxy + // path in front of the API). + const endpoint = "/v1/agents/connect" + if !strings.HasSuffix(u.Path, endpoint) { + u.Path = strings.TrimRight(u.Path, "/") + endpoint + } + return u.String(), nil +} + +// handshake sends Register and waits for RegisterAck. +func (c *Client) handshake(ctx context.Context, ws *websocket.Conn) (*agentproto.RegisterAck, error) { + reg := &agentproto.Register{ + AgentVersion: c.cfg.AgentVersion, + Hostname: c.cfg.Hostname, + Os: c.cfg.OS, + Arch: c.cfg.Arch, + DockerVersion: c.cfg.DockerVersion, + Capabilities: c.cfg.Capabilities, + LastKnownConfigVersion: c.configVersion.Load(), + } + env := agentproto.NewEnvelope() + env.Seq = c.seq.Add(1) + env.Payload = &agentproto.Envelope_Register{Register: reg} + raw, err := proto.Marshal(env) + if err != nil { + return nil, fmt.Errorf("marshal register: %w", err) + } + hCtx, cancel := context.WithTimeout(ctx, registerTimeout) + defer cancel() + if err := ws.Write(hCtx, websocket.MessageBinary, raw); err != nil { + return nil, fmt.Errorf("write register: %w", err) + } + typ, data, err := ws.Read(hCtx) + if err != nil { + return nil, fmt.Errorf("read register_ack: %w", err) + } + if typ != websocket.MessageBinary { + return nil, fmt.Errorf("register_ack must be binary, got %s", typ) + } + ackEnv := &agentproto.Envelope{} + if err := proto.Unmarshal(data, ackEnv); err != nil { + return nil, fmt.Errorf("unmarshal register_ack: %w", err) + } + p, ok := ackEnv.Payload.(*agentproto.Envelope_RegisterAck) + if !ok || p.RegisterAck == nil { + return nil, fmt.Errorf("first server payload must be RegisterAck, got %T", ackEnv.Payload) + } + return p.RegisterAck, nil +} + +// replayQueue dumps any pending RunBackup envelopes from the persistent +// queue onto the live out channel. Idempotent on the server side via +// run_id. Best-effort: a single failure stops replay but does not tear +// down the connection. +func (c *Client) replayQueue(ctx context.Context, out chan<- *agentproto.Envelope) { + if c.queue == nil { + return + } + jobs, err := c.queue.Pop(ctx, 100) + if err != nil { + c.logger.Warn("queue: pop failed", slog.Any("err", err)) + return + } + for _, j := range jobs { + env := &agentproto.Envelope{} + if err := proto.Unmarshal(j.Payload, env); err != nil { + c.logger.Warn("queue: corrupt payload; dropping", + slog.String("run_id", j.RunID), slog.Any("err", err)) + _ = c.queue.Ack(j.RunID) + continue + } + select { + case out <- env: + _ = c.queue.Ack(j.RunID) + case <-ctx.Done(): + return + } + } +} + +// osHostname is overridable in tests; defaults to os.Hostname. +var osHostname = func() (string, error) { + return hostname() +} diff --git a/apps/agent/internal/wss/client_test.go b/apps/agent/internal/wss/client_test.go new file mode 100644 index 0000000..7adbf10 --- /dev/null +++ b/apps/agent/internal/wss/client_test.go @@ -0,0 +1,178 @@ +package wss + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + agentproto "github.com/backupy/backupy/apps/agent/internal/proto" + "github.com/backupy/backupy/apps/agent/internal/queue" +) + +func TestBuildWSURL(t *testing.T) { + tests := []struct { + name string + in string + insecure bool + want string + expectErr bool + }{ + {"https rewrites to wss", "https://api.example.com", false, "wss://api.example.com/v1/agents/connect", false}, + {"http rejected without flag", "http://localhost:8080", false, "", true}, + {"http accepted with flag", "http://localhost:8080", true, "ws://localhost:8080/v1/agents/connect", false}, + {"already wss preserved", "wss://api.example.com", false, "wss://api.example.com/v1/agents/connect", false}, + {"path preserved", "https://api.example.com/proxy", false, "wss://api.example.com/proxy/v1/agents/connect", false}, + {"unknown scheme", "ftp://nope", false, "", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := buildWSURL(tc.in, tc.insecure) + if tc.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestClient_DispatchPingReturnsAck(t *testing.T) { + c := &Client{} + env := agentproto.NewEnvelope() + env.CorrelationId = "ping-1" + env.Payload = &agentproto.Envelope_Ping{Ping: &agentproto.Ping{TsMs: 123}} + // Send needs an out channel or queue; here we exercise the Send + // fallback path: with queue=nil and out=nil, Send returns an error + // that we intentionally swallow. Dispatch should not panic. + require.NotPanics(t, func() { + _ = c.dispatch(context.Background(), env) + }) +} + +// fakeServer is a minimal coder/websocket echo that performs a +// Register/RegisterAck handshake then records any inbound frames into a +// channel. Used to drive end-to-end client behaviour without spinning +// up the real server package. +func fakeServer(t *testing.T, inboundCh chan<- *agentproto.Envelope) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "Bearer test-key", r.Header.Get("Authorization")) + ws, err := websocket.Accept(w, r, &websocket.AcceptOptions{InsecureSkipVerify: true}) + require.NoError(t, err) + ctx := r.Context() + // Read Register. + _, data, err := ws.Read(ctx) + require.NoError(t, err) + env := &agentproto.Envelope{} + require.NoError(t, proto.Unmarshal(data, env)) + _, ok := env.Payload.(*agentproto.Envelope_Register) + require.True(t, ok) + // Send RegisterAck. + ack := agentproto.NewEnvelope() + ack.Payload = &agentproto.Envelope_RegisterAck{RegisterAck: &agentproto.RegisterAck{ + SessionId: "sess-1", HeartbeatIntervalSec: 30, + Config: &agentproto.AgentConfig{Version: 5}, + }} + raw, _ := proto.Marshal(ack) + require.NoError(t, ws.Write(ctx, websocket.MessageBinary, raw)) + + // Forward subsequent frames to inboundCh until the conn closes. + for { + _, data, err := ws.Read(ctx) + if err != nil { + return + } + env := &agentproto.Envelope{} + if proto.Unmarshal(data, env) != nil { + continue + } + select { + case inboundCh <- env: + default: + } + } + })) +} + +func TestClient_RegisterAndHeartbeat(t *testing.T) { + inbound := make(chan *agentproto.Envelope, 16) + ts := fakeServer(t, inbound) + defer ts.Close() + + var hbSeen atomic.Bool + c := NewClient(Config{ + ServerURL: ts.URL, + AgentKey: "test-key", + AgentVersion: "test", + AllowInsecure: true, + }, nil, nil, nil, func() *agentproto.AgentMetrics { + return &agentproto.AgentMetrics{CpuPercent: 1.5} + }, nil) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = c.Start(ctx) }() + + // Expect to see a Heartbeat in the inbound stream. + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + select { + case env := <-inbound: + if _, ok := env.Payload.(*agentproto.Envelope_Heartbeat); ok { + hbSeen.Store(true) + } + case <-time.After(100 * time.Millisecond): + } + if hbSeen.Load() { + break + } + } + require.True(t, hbSeen.Load(), "expected at least one Heartbeat envelope") + require.NotEmpty(t, c.SessionID()) + require.Equal(t, uint64(5), c.ConfigVersion()) +} + +func TestClient_SendWithoutConnectionQueues(t *testing.T) { + // Use a memory queue stub. + q := &memQueue{} + c := NewClient(Config{ + ServerURL: "wss://example.com", + AgentKey: "k", + AgentVersion: "v", + }, nil, q, nil, nil, nil) + + env := agentproto.NewEnvelope() + env.CorrelationId = "x-1" + env.Payload = &agentproto.Envelope_Heartbeat{Heartbeat: &agentproto.Heartbeat{}} + require.NoError(t, c.Send(env)) + require.Equal(t, 1, q.depth) +} + +// memQueue is a stub queue used by the test above. It satisfies the +// queue.Queue interface without bringing in BoltDB. +type memQueue struct { + depth int + last []byte +} + +func (m *memQueue) Enqueue(_ string, payload []byte) error { + m.depth++ + m.last = payload + return nil +} +func (m *memQueue) Pop(_ context.Context, _ int) ([]queue.Job, error) { return nil, nil } +func (m *memQueue) Ack(_ string) error { return nil } +func (m *memQueue) Depth() (int, error) { return m.depth, nil } + +// ensure strings is used to silence unused-import linter when tests +// shift around. +var _ = strings.TrimSpace diff --git a/apps/agent/internal/wss/loops.go b/apps/agent/internal/wss/loops.go new file mode 100644 index 0000000..acf54af --- /dev/null +++ b/apps/agent/internal/wss/loops.go @@ -0,0 +1,186 @@ +package wss + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "time" + + "github.com/coder/websocket" + "google.golang.org/protobuf/proto" + + agentproto "github.com/backupy/backupy/apps/agent/internal/proto" +) + +// readLoop pumps inbound frames from the server into the dispatch +// switch until the connection breaks or ctx is cancelled. +func (c *Client) readLoop(ctx context.Context, ws *websocket.Conn) error { + for { + if err := ctx.Err(); err != nil { + return err + } + readCtx, cancel := context.WithTimeout(ctx, readIdleTimeout) + typ, data, err := ws.Read(readCtx) + cancel() + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("read timeout after %s", readIdleTimeout) + } + return fmt.Errorf("read: %w", err) + } + if typ != websocket.MessageBinary { + c.logger.Warn("wss: ignoring non-binary frame", slog.String("type", typ.String())) + continue + } + env := &agentproto.Envelope{} + if err := proto.Unmarshal(data, env); err != nil { + c.logger.Warn("wss: unmarshal failed; closing", slog.Any("err", err)) + return fmt.Errorf("unmarshal: %w", err) + } + if err := c.dispatch(ctx, env); err != nil { + c.logger.Warn("wss: dispatch error", + slog.String("payload_type", payloadName(env)), + slog.Any("err", err)) + } + } +} + +// dispatch routes a single inbound envelope to a user callback. +func (c *Client) dispatch(ctx context.Context, env *agentproto.Envelope) error { + if env == nil || env.Payload == nil { + return errors.New("empty envelope") + } + switch p := env.Payload.(type) { + case *agentproto.Envelope_ConfigUpdate: + cfg := p.ConfigUpdate.GetConfig() + if cfg != nil { + if raw, err := proto.Marshal(cfg); err == nil && c.state != nil { + _ = c.state.SaveConfig(cfg.Version, raw) + } + c.configVersion.Store(cfg.Version) + } + if c.handlers != nil && c.handlers.OnConfigUpdate != nil { + return c.handlers.OnConfigUpdate(ctx, p.ConfigUpdate) + } + case *agentproto.Envelope_RunBackup: + if c.handlers != nil && c.handlers.OnRunBackup != nil { + return c.handlers.OnRunBackup(ctx, p.RunBackup) + } + case *agentproto.Envelope_CancelJob: + if c.handlers != nil && c.handlers.OnCancelJob != nil { + return c.handlers.OnCancelJob(ctx, p.CancelJob) + } + case *agentproto.Envelope_RunHealthCheck: + if c.handlers != nil && c.handlers.OnRunHealthCheck != nil { + return c.handlers.OnRunHealthCheck(ctx, p.RunHealthCheck) + } + case *agentproto.Envelope_SelfUpdate: + if c.handlers != nil && c.handlers.OnSelfUpdate != nil { + return c.handlers.OnSelfUpdate(ctx, p.SelfUpdate) + } + case *agentproto.Envelope_Ping: + // Reply with an Ack so the server has a round-trip metric. + ack := agentproto.NewEnvelope() + ack.CorrelationId = env.CorrelationId + ack.Payload = &agentproto.Envelope_Ack{Ack: &agentproto.Ack{ + CorrelationId: env.CorrelationId, Accepted: true, + }} + return c.Send(ack) + case *agentproto.Envelope_RegisterAck: + // Should only arrive during handshake; ignore here. + return nil + default: + return fmt.Errorf("unsupported server->agent payload %T", p) + } + return nil +} + +// writeLoop drains the per-connection out channel into the wire. +func (c *Client) writeLoop(ctx context.Context, ws *websocket.Conn, out <-chan *agentproto.Envelope) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case env := <-out: + if env == nil { + continue + } + if env.Seq == 0 { + env.Seq = c.seq.Add(1) + } + if env.TsMs == 0 { + env.TsMs = agentproto.NowMillis() + } + raw, err := proto.Marshal(env) + if err != nil { + c.logger.Warn("wss: marshal failed; dropping", + slog.String("payload_type", payloadName(env)), + slog.Any("err", err)) + continue + } + wCtx, cancel := context.WithTimeout(ctx, writeTimeout) + err = ws.Write(wCtx, websocket.MessageBinary, raw) + cancel() + if err != nil { + return fmt.Errorf("write: %w", err) + } + } + } +} + +// heartbeatLoop sends a Heartbeat envelope every heartbeatInterval. +func (c *Client) heartbeatLoop(ctx context.Context, out chan<- *agentproto.Envelope) error { + t := time.NewTicker(heartbeatInterval) + defer t.Stop() + // Send an initial heartbeat right after the handshake so the + // server's last_seen_at populates without waiting 30s. + if err := c.emitHeartbeat(ctx, out); err != nil { + return err + } + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + if err := c.emitHeartbeat(ctx, out); err != nil { + return err + } + } + } +} + +func (c *Client) emitHeartbeat(ctx context.Context, out chan<- *agentproto.Envelope) error { + hb := &agentproto.Heartbeat{ConfigVersion: c.configVersion.Load()} + if c.metrics != nil { + hb.Metrics = c.metrics() + } + env := agentproto.NewEnvelope() + env.Payload = &agentproto.Envelope_Heartbeat{Heartbeat: hb} + select { + case out <- env: + if c.state != nil { + _ = c.state.RecordHeartbeat(time.Now().UnixMilli()) + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// payloadName returns a short identifier for log lines. +func payloadName(env *agentproto.Envelope) string { + if env == nil || env.Payload == nil { + return "empty" + } + return fmt.Sprintf("%T", env.Payload) +} + +// hostname is a thin wrapper around os.Hostname so tests can stub it. +func hostname() (string, error) { + return os.Hostname() +} diff --git a/apps/backupy-decrypt/.gitkeep b/apps/backupy-decrypt/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/backupy-decrypt/.goreleaser.yaml b/apps/backupy-decrypt/.goreleaser.yaml new file mode 100644 index 0000000..8f84654 --- /dev/null +++ b/apps/backupy-decrypt/.goreleaser.yaml @@ -0,0 +1,53 @@ +# goreleaser config for backupy-decrypt — see goreleaser.com. +# Build with `goreleaser release --clean` once a tag is pushed. + +version: 2 + +project_name: backupy-decrypt + +before: + hooks: + - go mod tidy + +builds: + - id: backupy-decrypt + main: ./cmd/backupy-decrypt + binary: backupy-decrypt + env: + - CGO_ENABLED=0 + flags: + - -trimpath + ldflags: + - -s -w -X main.version={{.Version}} + goos: [linux, darwin, windows] + goarch: [amd64, arm64] + ignore: + - goos: windows + goarch: arm64 + +archives: + - id: default + formats: [tar.gz] + format_overrides: + - goos: windows + formats: [zip] + name_template: >- + {{ .ProjectName }}_{{ .Version }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + +checksum: + name_template: "checksums.txt" + +snapshot: + version_template: "{{ .Tag }}-next" + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" diff --git a/apps/backupy-decrypt/Makefile b/apps/backupy-decrypt/Makefile new file mode 100644 index 0000000..37dc4fc --- /dev/null +++ b/apps/backupy-decrypt/Makefile @@ -0,0 +1,37 @@ +# backupy-decrypt — Makefile +# +# Building: +# make build # builds for the host OS/arch into ./bin/ +# make release # multi-platform via goreleaser (requires goreleaser) +# make test # go test -race +# make tidy # go mod tidy +# make install # go install into $GOBIN +# +# Versioning: the binary's --version output is overridden via -ldflags. + +BINARY := backupy-decrypt +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +LDFLAGS := -s -w -X main.version=$(VERSION) +GO ?= go + +.PHONY: all build test tidy release install clean + +all: build + +build: + CGO_ENABLED=0 $(GO) build -trimpath -ldflags '$(LDFLAGS)' -o bin/$(BINARY) ./cmd/backupy-decrypt + +test: + $(GO) test -race -count=1 ./... + +tidy: + $(GO) mod tidy + +install: + CGO_ENABLED=0 $(GO) install -trimpath -ldflags '$(LDFLAGS)' ./cmd/backupy-decrypt + +release: + goreleaser release --clean --snapshot + +clean: + rm -rf bin dist diff --git a/apps/backupy-decrypt/README.md b/apps/backupy-decrypt/README.md new file mode 100644 index 0000000..d3b2c1b --- /dev/null +++ b/apps/backupy-decrypt/README.md @@ -0,0 +1,133 @@ +# backupy-decrypt + +Offline CLI to decrypt a [Backupy](https://backupy.ru) backup file. + +The Backupy server never sees your plaintext data — it stores only the +AES-256-GCM ciphertext your agent uploaded. To recover a backup you: + +1. Generate a **download URL** from the Backupy dashboard (`GET + /v1/runs/:id/download-url`). +2. Generate a **decryption token** from the same dashboard (`POST + /v1/runs/:id/decryption-token`). This token is a JWT that contains the + plaintext data-encryption key (DEK) plus integrity metadata. It is + valid for **15 minutes**. +3. Download the encrypted blob with `curl` (or any HTTP client). +4. Run `backupy-decrypt --token --in --out `. + +The CLI is fully offline — it never contacts Backupy infrastructure. + +## Install + +Download the release archive for your platform from the [GitHub +releases](https://github.com/backupy/backupy/releases) page, extract it, +and put the `backupy-decrypt` binary somewhere on `$PATH`: + +```bash +# macOS (arm64) — adjust for your platform +curl -L https://github.com/backupy/backupy/releases/latest/download/backupy-decrypt_Darwin_arm64.tar.gz \ + | tar -xz +sudo mv backupy-decrypt /usr/local/bin/ +``` + +Or build from source: + +```bash +git clone https://github.com/backupy/backupy.git +cd backupy/apps/backupy-decrypt +make build +./bin/backupy-decrypt --version +``` + +## Usage + +```bash +$ backupy-decrypt --help +Decrypt a Backupy backup file. + +Usage: + backupy-decrypt --token <jwt> --in <encrypted-file> --out <plaintext-file> + +Flags: + --token Decryption token (JWT) from the Backupy server + --token-file Read token from a file (avoids leaking via `ps`) + --in Encrypted input file (.enc) + --out Output plaintext file + --verify-sha256 Verify the input's SHA-256 matches the token's claim (default true) + --skip-decompress Don't decompress zstd after decryption (default false — the + agent always zstd-compresses, so leave this off unless your + pipeline used compression=none) + --quiet Suppress progress output + --version Show version +``` + +### Full example + +```bash +# 1) Download the ciphertext. +curl -o backup.enc "<presigned_url_from_dashboard>" + +# 2) Save the JWT into a file so it doesn't leak via your shell history. +echo "<jwt_from_dashboard>" > /tmp/backup.token + +# 3) Decrypt + decompress in one shot. Output is the raw pg_dump / mysqldump. +backupy-decrypt \ + --token-file /tmp/backup.token \ + --in backup.enc \ + --out backup.sql + +# 4) Restore yourself. Backupy intentionally does not run this step — +# you stay in control of where the data goes. +psql -d mydb < backup.sql +``` + +## Encryption format (verbatim) + +The CLI implements the inverse of `apps/agent/internal/pipeline`. All +integers are **big-endian**. + +``` +chunk := uint32 ciphertext_len // bytes that follow, EXCLUDING this u32 + 12-byte random nonce // unique per chunk + ciphertext (≤ 1 MiB + 16-byte GCM tag) + +EOF marker := uint32 0 // appended after the final chunk +``` + +- Plaintext chunk size is **1 MiB**. The final chunk may be shorter. +- `tag` is the 16-byte AES-GCM authentication tag, appended by Go's + `cipher.AEAD.Seal`. +- The Additional Authenticated Data (AAD) is **nil**. Per-chunk reorder + and truncation defence comes from the explicit `uint32` size prefix + plus the mandatory zero-length EOF marker: any reordered or truncated + stream either trips a frame-boundary error or fails the missing-EOF + check. +- The DEK is exactly **32 bytes** (AES-256). + +This format is canonical — `apps/agent/internal/pipeline/encrypt.go` is +the single source of truth, and the CLI is implemented to be byte-for- +byte compatible. This is enough to reimplement the decrypt step in any +language — see the package doc for `apps/backupy-decrypt/internal/decrypt`. + +## Troubleshooting + +| Symptom | Cause / fix | +|--------------------------------------------------------|------------------------------------------------------------------------------------------------------| +| `error: decrypt: token expired (request a new one)` | The JWT lifetime is 15 min. Issue a new token from the dashboard. | +| `error: decrypt: AES-GCM authentication failed` | Wrong DEK or corrupted input. Re-download the file and re-issue the token. | +| `error: decrypt: ciphertext SHA-256 mismatch` | The file you downloaded is not the file the server recorded. Re-download — the URL may have expired. | +| `error: decrypt: input file is truncated` | Download didn't complete. Retry with `curl -C - ...` or a fresh URL. | +| Plaintext looks like binary garbage after decompress | Your job was set to `compression=none`. Re-run with `--skip-decompress`. | + +## Security notes + +- The decryption token is **as sensitive as your data** — anyone with the + JWT in their possession can decrypt the file. Treat it like a temporary + password: do not paste it into chat, do not commit it. +- The CLI never writes the DEK to disk; it lives only in process memory + and is zeroized before exit. +- For an extra layer of safety pass `--token-file` so the token does not + appear in your shell history or in `ps`. + +## License + +Same as the parent Backupy repository. diff --git a/apps/backupy-decrypt/cmd/backupy-decrypt/main.go b/apps/backupy-decrypt/cmd/backupy-decrypt/main.go new file mode 100644 index 0000000..916414c --- /dev/null +++ b/apps/backupy-decrypt/cmd/backupy-decrypt/main.go @@ -0,0 +1,135 @@ +// Command backupy-decrypt decrypts a Backupy backup file produced by the +// Backupy agent. +// +// The CLI takes a JWT (issued by the Backupy server at +// /v1/runs/:id/decryption-token) and an encrypted input file. It streams +// the input through AES-256-GCM decryption, optionally decompresses zstd, +// and writes the plaintext to the chosen output path. +// +// The CLI is OFFLINE-ONLY: it never contacts the server. The JWT carries +// everything we need (the DEK is embedded in the token body). +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/backupy/backupy/apps/backupy-decrypt/internal/decrypt" +) + +// version is overwritten at build time via -ldflags "-X main.version=...". +var version = "dev" + +func main() { + var ( + token = flag.String("token", "", "Decryption token (JWT) from the Backupy server.") + input = flag.String("in", "", "Encrypted input file (.enc).") + output = flag.String("out", "", "Output plaintext file.") + verifySHA = flag.Bool("verify-sha256", true, "Verify that the input's SHA-256 matches the token's claim.") + skipDecomp = flag.Bool("skip-decompress", false, "Don't decompress zstd after decrypting (the agent always zstd-compresses, so this only helps if you have a non-default pipeline).") + quiet = flag.Bool("quiet", false, "Suppress progress output.") + showVersion = flag.Bool("version", false, "Show version and exit.") + flagTokenFile = flag.String("token-file", "", "Read token from this file instead of --token (useful to avoid leaking via ps).") + ) + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "Decrypt a Backupy backup file.") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, "Usage:") + fmt.Fprintln(os.Stderr, " backupy-decrypt --token <jwt> --in <encrypted-file> --out <plaintext-file>") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, "Flags:") + flag.PrintDefaults() + } + flag.Parse() + + if *showVersion { + fmt.Printf("backupy-decrypt %s\n", version) + return + } + + tok := *token + if *flagTokenFile != "" { + b, err := os.ReadFile(*flagTokenFile) + if err != nil { + fail("read --token-file: %v", err) + } + tok = trimNewline(string(b)) + } + + if tok == "" || *input == "" || *output == "" { + flag.Usage() + os.Exit(2) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + progress := func(n int64) {} + if !*quiet { + progress = makeProgressFn() + } + + if err := decrypt.Run(ctx, decrypt.Options{ + InputPath: *input, + OutputPath: *output, + Token: tok, + VerifySHA256: *verifySHA, + SkipDecompress: *skipDecomp, + Progress: progress, + }); err != nil { + fail("%v", err) + } + + if !*quiet { + fmt.Fprintln(os.Stderr, "OK — decrypted to", *output) + } +} + +func fail(format string, args ...any) { + fmt.Fprintf(os.Stderr, "error: "+format+"\n", args...) + os.Exit(1) +} + +func trimNewline(s string) string { + for len(s) > 0 && (s[len(s)-1] == '\n' || s[len(s)-1] == '\r' || s[len(s)-1] == ' ' || s[len(s)-1] == '\t') { + s = s[:len(s)-1] + } + return s +} + +// makeProgressFn returns a Progress callback that overwrites a single +// line on stderr with the byte count. Outputs a final newline when the +// caller stops calling it (best-effort). +func makeProgressFn() func(int64) { + var last int64 + return func(n int64) { + // Only redraw on ~1 MiB granularity to avoid spamming stderr. + if n-last < 1<<20 { + return + } + last = n + fmt.Fprintf(os.Stderr, "\rprocessed %s", humanBytes(n)) + } +} + +func humanBytes(n int64) string { + const ( + KB = 1 << 10 + MB = 1 << 20 + GB = 1 << 30 + ) + switch { + case n >= GB: + return fmt.Sprintf("%.2f GiB", float64(n)/GB) + case n >= MB: + return fmt.Sprintf("%.2f MiB", float64(n)/MB) + case n >= KB: + return fmt.Sprintf("%.2f KiB", float64(n)/KB) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/apps/backupy-decrypt/doc.go b/apps/backupy-decrypt/doc.go new file mode 100644 index 0000000..d852b12 --- /dev/null +++ b/apps/backupy-decrypt/doc.go @@ -0,0 +1,8 @@ +// Package backupydecrypt is the root of the offline backup-decryption CLI. +// +// The compiled binary lives under cmd/backupy-decrypt; the reusable +// streaming decrypt logic is in internal/decrypt. This file exists so +// that integration tests at the module root (e.g. integration_test.go, +// guarded by the `integration` build tag) have a real package to attach +// to — Go requires at least one non-test file per directory. +package backupydecrypt diff --git a/apps/backupy-decrypt/go.mod b/apps/backupy-decrypt/go.mod new file mode 100644 index 0000000..0ae81aa --- /dev/null +++ b/apps/backupy-decrypt/go.mod @@ -0,0 +1,31 @@ +module github.com/backupy/backupy/apps/backupy-decrypt + +go 1.22 + +require ( + github.com/klauspost/compress v1.17.9 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +// Integration tests (build tag: integration) cross-link to the agent's +// pipeline package to prove the encrypt/decrypt formats match byte-for-byte. +// The replace directives point at local sibling modules; the requires +// use the zero-version pseudo-tag that go-mod accepts for replace targets. +// The proto/gen replace must be declared here too because Go's module +// resolution does not transitively honour the agent module's own replace. +require ( + github.com/backupy/backupy/apps/agent v0.0.0-00010101000000-000000000000 + github.com/backupy/backupy/packages/proto/gen/go/backupv1 v0.0.0-00010101000000-000000000000 // indirect +) + +replace ( + github.com/backupy/backupy/apps/agent => ../agent + github.com/backupy/backupy/packages/proto/gen/go/backupv1 => ../../packages/proto/gen/go/backupv1 +) diff --git a/apps/backupy-decrypt/go.sum b/apps/backupy-decrypt/go.sum new file mode 100644 index 0000000..0a8dbd4 --- /dev/null +++ b/apps/backupy-decrypt/go.sum @@ -0,0 +1,18 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/backupy-decrypt/integration_test.go b/apps/backupy-decrypt/integration_test.go new file mode 100644 index 0000000..993befd --- /dev/null +++ b/apps/backupy-decrypt/integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +// Cross-binary integration test: prove the agent's pipeline.Encryptor +// produces a byte stream that the backupy-decrypt CLI can consume. +// +// This is the GOLDEN TEST for Phase 1. If this passes, the entire +// backup → download → decrypt loop works end-to-end at the byte level. +// +// Run via: +// +// cd apps/backupy-decrypt && go test -tags=integration ./... +// +// The test depends on the agent module via a `replace` directive in +// go.mod, which in turn depends on the generated proto bindings. So +// `make proto` must have been run from the repo root first. +package backupydecrypt_test + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/backupy/backupy/apps/agent/internal/pipeline" + "github.com/backupy/backupy/apps/backupy-decrypt/internal/decrypt" +) + +// craftToken builds a JWT in the shape the CLI expects. The signature +// is ignored — the CLI verifies neither HMAC nor RSA, only the claim +// envelope and the SHA-256 inside it. +func craftToken(t *testing.T, dek []byte, sha string) string { + t.Helper() + hdr := map[string]string{"alg": "HS256", "typ": "JWT"} + hdrJSON, err := json.Marshal(hdr) + require.NoError(t, err) + claims := map[string]any{ + "iss": "backupy-server", + "sub": "user-1", + "aud": "backupy-decrypt", + "iat": time.Now().Unix(), + "exp": time.Now().Add(15 * time.Minute).Unix(), + "run_id": "run-integration", + "company_id": "co-integration", + "dek": base64.StdEncoding.EncodeToString(dek), + "alg": "AES-256-GCM", + "format_version": 1, + "sha256": sha, + } + pld, err := json.Marshal(claims) + require.NoError(t, err) + enc := base64.RawURLEncoding + signingInput := enc.EncodeToString(hdrJSON) + "." + enc.EncodeToString(pld) + return signingInput + "." + enc.EncodeToString([]byte("ignored")) +} + +// TestCrossBinary_PipelineToDecrypt is the bytewise round-trip check. +func TestCrossBinary_PipelineToDecrypt(t *testing.T) { + cases := []struct { + name string + size int + }{ + {"small_46B", 46}, + {"single_full_chunk_1MiB", pipeline.ChunkPlainSize}, + {"two_chunks_plus_remainder", 2*pipeline.ChunkPlainSize + 1234}, + {"empty", 0}, + } + // Compile-time sanity: the two sides MUST agree on the chunk size. + require.Equal(t, pipeline.ChunkPlainSize, decrypt.ChunkPlaintextSize, + "chunk size constant drift between pipeline and decrypt — re-sync") + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + plaintext := make([]byte, tc.size) + _, err := rand.Read(plaintext) + require.NoError(t, err) + + dek := make([]byte, 32) + _, err = rand.Read(dek) + require.NoError(t, err) + + // Encrypt with the agent's pipeline. + enc, err := pipeline.NewEncryptor(dek) + require.NoError(t, err) + var ctBuf bytes.Buffer + n, err := enc.Stream(bytes.NewReader(plaintext), &ctBuf) + require.NoError(t, err) + require.Equal(t, int64(len(plaintext)), n) + ciphertext := ctBuf.Bytes() + + // SHA-256 of the on-wire bytes — what the JWT carries. + sum := sha256.Sum256(ciphertext) + shaHex := hex.EncodeToString(sum[:]) + + // Write to disk, decrypt with the CLI's package. + dir := t.TempDir() + inPath := filepath.Join(dir, "in.enc") + outPath := filepath.Join(dir, "out.bin") + require.NoError(t, os.WriteFile(inPath, ciphertext, 0o600)) + + tok := craftToken(t, dek, shaHex) + err = decrypt.Run(context.Background(), decrypt.Options{ + InputPath: inPath, + OutputPath: outPath, + Token: tok, + VerifySHA256: true, + SkipDecompress: true, + }) + require.NoError(t, err, "decrypt.Run must accept pipeline output") + + got, err := os.ReadFile(outPath) + require.NoError(t, err) + require.Equal(t, plaintext, got, "round-trip must be bytewise identical") + }) + } +} diff --git a/apps/backupy-decrypt/internal/decrypt/decrypt.go b/apps/backupy-decrypt/internal/decrypt/decrypt.go new file mode 100644 index 0000000..a4e5eb9 --- /dev/null +++ b/apps/backupy-decrypt/internal/decrypt/decrypt.go @@ -0,0 +1,321 @@ +// Package decrypt streams a Backupy backup file through: +// +// 1. AES-256-GCM decryption (chunked frames, see Wire format below) +// 2. zstd decompression +// 3. SHA-256 verification of the ciphertext +// +// Wire format (verbatim mirror of apps/agent/internal/pipeline/encrypt.go — +// that file is the single source of truth). All integers big-endian: +// +// chunk := uint32 ciphertext_len // bytes that follow, EXCLUDING this u32 +// 12-byte random nonce // unique per chunk +// ciphertext (≤ ChunkPlaintextSize + 16-byte GCM tag) +// +// EOF marker: a single chunk with ciphertext_len == 0. The decryptor +// treats this as "stream finished cleanly" — without it, a truncated +// upload would be indistinguishable from a clean end. +// +// Chunk plaintext size is ChunkPlaintextSize = 1 MiB. The AEAD's +// Additional Authenticated Data is nil — chunk reorder/replay defence +// is provided by the EOF marker + explicit per-chunk size prefix: +// any reorder breaks the frame boundaries and fails the next length +// read, and any truncated tail trips the missing-EOF check. +// +// The CLI verifies the SHA-256 of all bytes read from the input file +// (i.e. the concatenation of every frame: u32 size + nonce + ct+tag, +// including the trailing zero EOF marker) against the JWT's "sha256" +// claim before declaring success. +// +// All operations are streaming — we never materialise a full plaintext +// or ciphertext block beyond ChunkPlaintextSize+overhead. +package decrypt + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/klauspost/compress/zstd" + + "github.com/backupy/backupy/apps/backupy-decrypt/internal/jwt" +) + +// Constants — pipeline contract. Keep in sync with +// apps/agent/internal/pipeline/encrypt.go. +const ( + // ChunkPlaintextSize is the maximum size of one plaintext chunk + // before encryption. The pipeline ships exactly 1 MiB per chunk; the + // final chunk may be shorter. MUST equal pipeline.ChunkPlainSize. + ChunkPlaintextSize = 1 << 20 // 1 MiB + + // NonceSize is GCM's 96-bit nonce. + NonceSize = 12 + + // TagSize is GCM's 128-bit auth tag. + TagSize = 16 + + // ChunkHeaderSize is the 4-byte big-endian length prefix per chunk. + ChunkHeaderSize = 4 + + // IssuerExpected is the iss claim we accept on JWTs. + IssuerExpected = "backupy-server" + + // AudienceExpected is the aud claim we accept on JWTs. + AudienceExpected = "backupy-decrypt" +) + +// Errors callers should care about. +var ( + ErrTokenExpired = errors.New("decrypt: token expired (request a new one)") + ErrInvalidToken = errors.New("decrypt: invalid token") + ErrSHA256Mismatch = errors.New("decrypt: ciphertext SHA-256 mismatch — file is corrupt or token is for a different run") + ErrTruncated = errors.New("decrypt: input file is truncated") + ErrDecryptFailed = errors.New("decrypt: AES-GCM authentication failed — wrong key or corrupted data") + ErrUnsupportedAlg = errors.New("decrypt: unsupported algorithm") + ErrUnsupportedFmt = errors.New("decrypt: unsupported format version") + ErrFrameTooLarge = errors.New("decrypt: frame size exceeds maximum") + ErrFrameTooSmall = errors.New("decrypt: frame size below minimum") +) + +// Options controls a single decrypt run. +type Options struct { + InputPath string + OutputPath string + Token string // JWT + VerifySHA256 bool + SkipDecompress bool + // Progress is called periodically with the count of input bytes + // consumed so far. Optional. + Progress func(bytesProcessed int64) +} + +// Run executes the decrypt + (optional) decompress pipeline. +func Run(ctx context.Context, opts Options) error { + claims, err := jwt.ParseDecryption(opts.Token, IssuerExpected, AudienceExpected) + if err != nil { + switch { + case errors.Is(err, jwt.ErrExpired): + return fmt.Errorf("%w: %v. Request a new one from the Backupy dashboard.", ErrTokenExpired, err) + default: + return fmt.Errorf("%w: %v", ErrInvalidToken, err) + } + } + if !strings.EqualFold(claims.Algorithm, "AES-256-GCM") { + return fmt.Errorf("%w: %q", ErrUnsupportedAlg, claims.Algorithm) + } + if claims.FormatVersion != 1 { + return fmt.Errorf("%w: %d", ErrUnsupportedFmt, claims.FormatVersion) + } + + dek, err := base64.StdEncoding.DecodeString(claims.DEKBase64) + if err != nil { + return fmt.Errorf("%w: dek not valid base64: %v", ErrInvalidToken, err) + } + if len(dek) != 32 { + return fmt.Errorf("%w: dek length = %d, want 32", ErrInvalidToken, len(dek)) + } + defer zeroize(dek) + + in, err := os.Open(opts.InputPath) + if err != nil { + return fmt.Errorf("open input: %w", err) + } + defer in.Close() + + out, err := os.OpenFile(opts.OutputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("open output: %w", err) + } + outClosed := false + defer func() { + if !outClosed { + _ = out.Close() + } + }() + + block, err := aes.NewCipher(dek) + if err != nil { + return fmt.Errorf("aes: %w", err) + } + aead, err := cipher.NewGCM(block) + if err != nil { + return fmt.Errorf("gcm: %w", err) + } + + hasher := sha256.New() + // teeReader: every byte read from `in` is mirrored to the hasher, + // so the ciphertext SHA we compare against the JWT is over the full + // on-wire stream (size prefixes + nonces + ciphertexts + tags + + // EOF marker), matching what the agent computed. + source := io.TeeReader(in, hasher) + + // plaintextSink: either the file directly (SkipDecompress) or via a + // zstd decoder. We wrap the file in a NopCloser-equivalent so we can + // Close the sink uniformly without double-closing `out`. + var plaintextSink io.WriteCloser + if opts.SkipDecompress { + plaintextSink = noopCloser{out} + } else { + // We write *ciphertext-decrypted plaintext* into the zstd + // decoder's input. The decoder writes decompressed bytes to + // `out`. So we need an io.PipeWriter feeding into zstd.NewReader. + pr, pw := io.Pipe() + dec, err := zstd.NewReader(pr) + if err != nil { + _ = pr.Close() + _ = pw.Close() + return fmt.Errorf("zstd: %w", err) + } + // Run the decode -> out copy in a goroutine. + errCh := make(chan error, 1) + go func() { + _, copyErr := io.Copy(out, dec) + dec.Close() + errCh <- copyErr + }() + plaintextSink = &pipeWriterCloser{pw: pw, errCh: errCh} + } + + // Decrypt loop. + if err := decryptStream(ctx, source, plaintextSink, aead, opts.Progress); err != nil { + _ = plaintextSink.Close() + return err + } + if err := plaintextSink.Close(); err != nil { + return fmt.Errorf("close plaintext sink: %w", err) + } + // Flush the underlying file. Doing this here (rather than in defer) + // lets us return the close error if it happens (e.g. disk full at + // the very last fsync). + if err := out.Close(); err != nil { + outClosed = true + return fmt.Errorf("close output: %w", err) + } + outClosed = true + + if opts.VerifySHA256 && claims.SHA256 != "" { + got := hex.EncodeToString(hasher.Sum(nil)) + if !strings.EqualFold(got, claims.SHA256) { + return fmt.Errorf("%w: got %s, want %s", ErrSHA256Mismatch, got, claims.SHA256) + } + } + return nil +} + +// noopCloser is an io.WriteCloser whose Close is a no-op. Used so we can +// uniformly call Close on plaintextSink without double-closing the +// underlying os.File (Run closes it explicitly). +type noopCloser struct{ io.Writer } + +func (noopCloser) Close() error { return nil } + +// pipeWriterCloser bundles a pipe writer with the goroutine that drains +// the other side, so Close() waits for the drain to finish and returns +// its error. +type pipeWriterCloser struct { + pw *io.PipeWriter + errCh chan error +} + +func (p *pipeWriterCloser) Write(b []byte) (int, error) { return p.pw.Write(b) } +func (p *pipeWriterCloser) Close() error { + if err := p.pw.Close(); err != nil { + return err + } + return <-p.errCh +} + +// maxFrameSize is the upper bound on a single chunk's ciphertext_len. +// Anything bigger is rejected before we allocate — protects against a +// malicious file that claims a multi-gigabyte frame size. +const maxFrameSize = NonceSize + ChunkPlaintextSize + TagSize + +// decryptStream reads length-prefixed AES-GCM frames from r, writes the +// decrypted plaintext to w. The frame format is described in the package +// docstring. AAD is nil; per-chunk reorder defence is provided by the +// explicit size prefix + mandatory zero-length EOF marker. +func decryptStream(ctx context.Context, r io.Reader, w io.Writer, aead cipher.AEAD, progress func(int64)) error { + header := make([]byte, ChunkHeaderSize) + // Pre-allocate the maximum-size frame buffer once; reused across loops. + frameBuf := make([]byte, maxFrameSize) + var processed int64 + + for { + if err := ctx.Err(); err != nil { + return err + } + + // Read the 4-byte big-endian length prefix. + if _, err := io.ReadFull(r, header); err != nil { + if errors.Is(err, io.EOF) { + // Stream ended without an explicit terminator — refuse. + // The pipeline ALWAYS writes the zero-length EOF marker; + // missing one means the file was truncated mid-upload. + return fmt.Errorf("%w (no EOF marker)", ErrTruncated) + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("%w (chunk header)", ErrTruncated) + } + return fmt.Errorf("read chunk header: %w", err) + } + size := binary.BigEndian.Uint32(header) + processed += int64(ChunkHeaderSize) + + // EOF marker — clean end of stream. The pipeline writes this as + // the very last bytes of the upload. + if size == 0 { + if progress != nil { + progress(processed) + } + return nil + } + // Minimum frame is nonce + tag (i.e. encrypting zero plaintext). + if size < uint32(NonceSize+TagSize) { + return fmt.Errorf("%w: %d < %d", ErrFrameTooSmall, size, NonceSize+TagSize) + } + if size > uint32(maxFrameSize) { + return fmt.Errorf("%w: %d > %d", ErrFrameTooLarge, size, maxFrameSize) + } + + // Read the exact-size frame. + frame := frameBuf[:size] + if _, err := io.ReadFull(r, frame); err != nil { + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + return fmt.Errorf("%w (chunk body)", ErrTruncated) + } + return fmt.Errorf("read chunk body: %w", err) + } + nonce := frame[:NonceSize] + ct := frame[NonceSize:] + + // AAD is nil to match pipeline.Encryptor.Stream's seal call. + pt, err := aead.Open(nil, nonce, ct, nil) + if err != nil { + return fmt.Errorf("%w: %v", ErrDecryptFailed, err) + } + if _, err := w.Write(pt); err != nil { + return fmt.Errorf("write plaintext: %w", err) + } + + processed += int64(size) + if progress != nil { + progress(processed) + } + } +} + +// zeroize overwrites b in place. +func zeroize(b []byte) { + for i := range b { + b[i] = 0 + } +} diff --git a/apps/backupy-decrypt/internal/decrypt/decrypt_test.go b/apps/backupy-decrypt/internal/decrypt/decrypt_test.go new file mode 100644 index 0000000..481be1e --- /dev/null +++ b/apps/backupy-decrypt/internal/decrypt/decrypt_test.go @@ -0,0 +1,333 @@ +package decrypt + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/require" +) + +// encryptFixture mirrors the agent pipeline's encryption format so tests +// can produce input files that the CLI must decrypt successfully. +// +// MUST match decryptStream's layout. The pipeline's wire format is: +// +// repeated: +// uint32 big-endian ciphertext_len (= nonce + ct + tag bytes) +// 12 bytes random nonce +// ct+tag AES-256-GCM Seal output (AAD = nil) +// terminator: +// uint32 big-endian 0 +// +// The final plaintext chunk may be shorter than ChunkPlaintextSize. +func encryptFixture(t *testing.T, dek []byte, plaintext []byte) []byte { + t.Helper() + block, err := aes.NewCipher(dek) + require.NoError(t, err) + aead, err := cipher.NewGCM(block) + require.NoError(t, err) + + var out []byte + header := make([]byte, ChunkHeaderSize) + + off := 0 + for off < len(plaintext) { + end := off + ChunkPlaintextSize + if end > len(plaintext) { + end = len(plaintext) + } + chunk := plaintext[off:end] + nonce := make([]byte, NonceSize) + _, err := rand.Read(nonce) + require.NoError(t, err) + ct := aead.Seal(nil, nonce, chunk, nil) + binary.BigEndian.PutUint32(header, uint32(len(nonce)+len(ct))) + out = append(out, header...) + out = append(out, nonce...) + out = append(out, ct...) + off = end + } + // EOF marker — always present, even for empty plaintext. + binary.BigEndian.PutUint32(header, 0) + out = append(out, header...) + return out +} + +// zstdCompress is the agent-side compression step. +func zstdCompress(t *testing.T, plaintext []byte) []byte { + t.Helper() + enc, err := zstd.NewWriter(nil) + require.NoError(t, err) + defer enc.Close() + return enc.EncodeAll(plaintext, nil) +} + +// craftToken builds a JWT identical in shape to what the server issues, +// so the CLI accepts it. +func craftToken(t *testing.T, dek []byte, sha string) string { + t.Helper() + hdr := map[string]string{"alg": "HS256", "typ": "JWT"} + hdrJSON, err := json.Marshal(hdr) + require.NoError(t, err) + claims := map[string]any{ + "iss": "backupy-server", + "sub": "user-1", + "aud": "backupy-decrypt", + "iat": time.Now().Unix(), + "exp": time.Now().Add(15 * time.Minute).Unix(), + "run_id": "run-1", + "company_id": "co-1", + "dek": base64.StdEncoding.EncodeToString(dek), + "alg": "AES-256-GCM", + "format_version": 1, + "sha256": sha, + } + pld, err := json.Marshal(claims) + require.NoError(t, err) + enc := base64.RawURLEncoding + signingInput := enc.EncodeToString(hdrJSON) + "." + enc.EncodeToString(pld) + // Signature value doesn't matter — CLI doesn't verify HMAC. + return signingInput + "." + enc.EncodeToString([]byte("ignored")) +} + +func writeFile(t *testing.T, dir, name string, data []byte) string { + t.Helper() + p := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(p, data, 0o600)) + return p +} + +func TestDecrypt_RoundTrip_Compressed(t *testing.T) { + dir := t.TempDir() + plaintext := []byte("Hello, Backupy! This is a small test backup.\n") + compressed := zstdCompress(t, plaintext) + + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, compressed) + sum := sha256.Sum256(encrypted) + sha := hex.EncodeToString(sum[:]) + + in := writeFile(t, dir, "backup.enc", encrypted) + out := filepath.Join(dir, "backup.sql") + tok := craftToken(t, dek, sha) + + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + VerifySHA256: true, + }) + require.NoError(t, err) + + got, err := os.ReadFile(out) + require.NoError(t, err) + require.Equal(t, plaintext, got) +} + +func TestDecrypt_RoundTrip_MultiChunk(t *testing.T) { + dir := t.TempDir() + // 3 full chunks + a small remainder. + plaintext := make([]byte, ChunkPlaintextSize*3+1234) + _, _ = rand.Read(plaintext) + + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, plaintext) + sum := sha256.Sum256(encrypted) + + in := writeFile(t, dir, "backup.enc", encrypted) + out := filepath.Join(dir, "backup.bin") + tok := craftToken(t, dek, hex.EncodeToString(sum[:])) + + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + VerifySHA256: true, + SkipDecompress: true, + }) + require.NoError(t, err) + + got, err := os.ReadFile(out) + require.NoError(t, err) + require.Equal(t, plaintext, got) +} + +func TestDecrypt_RoundTrip_EmptyPlaintext(t *testing.T) { + // Even an empty payload must produce a valid stream: just the EOF + // marker. Round-tripping it must yield no plaintext and no error. + dir := t.TempDir() + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, nil) + sum := sha256.Sum256(encrypted) + + in := writeFile(t, dir, "empty.enc", encrypted) + out := filepath.Join(dir, "empty.out") + tok := craftToken(t, dek, hex.EncodeToString(sum[:])) + + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + VerifySHA256: true, + SkipDecompress: true, + }) + require.NoError(t, err) + + got, err := os.ReadFile(out) + require.NoError(t, err) + require.Empty(t, got) +} + +func TestDecrypt_WrongDEK(t *testing.T) { + dir := t.TempDir() + plaintext := []byte("secret stuff") + correctDEK := make([]byte, 32) + _, _ = rand.Read(correctDEK) + encrypted := encryptFixture(t, correctDEK, plaintext) + in := writeFile(t, dir, "x.enc", encrypted) + out := filepath.Join(dir, "x.out") + + wrongDEK := make([]byte, 32) + _, _ = rand.Read(wrongDEK) + tok := craftToken(t, wrongDEK, "") + + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + SkipDecompress: true, + }) + require.Error(t, err) + require.ErrorIs(t, err, ErrDecryptFailed) +} + +func TestDecrypt_Truncated(t *testing.T) { + dir := t.TempDir() + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, []byte("some data here")) + // Drop the trailing EOF marker (last 4 bytes). + bad := encrypted[:len(encrypted)-4] + in := writeFile(t, dir, "x.enc", bad) + out := filepath.Join(dir, "x.out") + + tok := craftToken(t, dek, "") + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + SkipDecompress: true, + }) + require.Error(t, err) + require.ErrorIs(t, err, ErrTruncated) +} + +func TestDecrypt_Truncated_MidFrame(t *testing.T) { + dir := t.TempDir() + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, []byte("some data here")) + // Truncate inside the first frame — even the nonce isn't complete. + bad := encrypted[:ChunkHeaderSize+5] + in := writeFile(t, dir, "x.enc", bad) + out := filepath.Join(dir, "x.out") + + tok := craftToken(t, dek, "") + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + SkipDecompress: true, + }) + require.Error(t, err) + require.ErrorIs(t, err, ErrTruncated) +} + +func TestDecrypt_SHA256Mismatch(t *testing.T) { + dir := t.TempDir() + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, []byte("contents")) + in := writeFile(t, dir, "x.enc", encrypted) + out := filepath.Join(dir, "x.out") + + // Use a deliberately wrong sha. + tok := craftToken(t, dek, "deadbeef") + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + VerifySHA256: true, + SkipDecompress: true, + }) + require.Error(t, err) + require.ErrorIs(t, err, ErrSHA256Mismatch) +} + +func TestDecrypt_ExpiredToken(t *testing.T) { + dir := t.TempDir() + dek := make([]byte, 32) + _, _ = rand.Read(dek) + // Build a JWT with exp in the past. + hdr := map[string]string{"alg": "HS256", "typ": "JWT"} + hdrJSON, _ := json.Marshal(hdr) + cl := map[string]any{ + "iss": "backupy-server", + "aud": "backupy-decrypt", + "sub": "u", + "iat": time.Now().Add(-time.Hour).Unix(), + "exp": time.Now().Add(-time.Minute).Unix(), + "run_id": "r", + "company_id": "c", + "dek": base64.StdEncoding.EncodeToString(dek), + "alg": "AES-256-GCM", + "format_version": 1, + } + pld, _ := json.Marshal(cl) + enc := base64.RawURLEncoding + tok := enc.EncodeToString(hdrJSON) + "." + enc.EncodeToString(pld) + "." + enc.EncodeToString([]byte("sig")) + + in := writeFile(t, dir, "x.enc", []byte("anything")) + out := filepath.Join(dir, "x.out") + err := Run(context.Background(), Options{ + InputPath: in, + OutputPath: out, + Token: tok, + SkipDecompress: true, + }) + require.Error(t, err) + require.ErrorIs(t, err, ErrTokenExpired) +} + +func TestDecrypt_ContextCancel(t *testing.T) { + // Make a large fake encrypted stream and cancel before reading. + dir := t.TempDir() + plaintext := make([]byte, ChunkPlaintextSize*4) + dek := make([]byte, 32) + _, _ = rand.Read(dek) + encrypted := encryptFixture(t, dek, plaintext) + in := writeFile(t, dir, "x.enc", encrypted) + out := filepath.Join(dir, "x.out") + tok := craftToken(t, dek, "") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := Run(ctx, Options{InputPath: in, OutputPath: out, Token: tok, SkipDecompress: true}) + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/apps/backupy-decrypt/internal/jwt/jwt.go b/apps/backupy-decrypt/internal/jwt/jwt.go new file mode 100644 index 0000000..ae3e15b --- /dev/null +++ b/apps/backupy-decrypt/internal/jwt/jwt.go @@ -0,0 +1,171 @@ +// Package jwt parses and validates JWTs issued by the Backupy server +// without verifying the HMAC signature. +// +// Rationale: the CLI cannot know the server's HS256 signing secret, so +// any signature check would be theatre. What we DO check is the token +// structure, the issuer/audience claims, and expiry — that's enough to +// produce a clear error message if the user pastes the wrong string. +// +// If a future server release ships a public-key signature scheme we can +// move to full crypto verification here. For now this is intentionally +// minimal — see apps/server/internal/jwt for the signing side. +package jwt + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// Sentinel errors. +var ( + ErrMalformed = errors.New("jwt: malformed token") + ErrExpired = errors.New("jwt: token expired") + ErrWrongIssuer = errors.New("jwt: wrong issuer") + ErrWrongAudience = errors.New("jwt: wrong audience") + ErrMissingClaim = errors.New("jwt: missing claim") +) + +// DecryptionClaims is the typed view of a decryption-token JWT. +type DecryptionClaims struct { + Issuer string + Subject string + Audience string + IssuedAt time.Time + ExpiresAt time.Time + RunID string + CompanyID string + DEKBase64 string + Algorithm string + FormatVersion int + SHA256 string +} + +// Parse decodes a JWT without verifying the HMAC signature, returns the +// raw claims map. ErrMalformed is returned on any structural failure. +func Parse(token string) (map[string]any, error) { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return nil, fmt.Errorf("%w: want 3 segments, got %d", ErrMalformed, len(parts)) + } + enc := base64.RawURLEncoding + pld, err := enc.DecodeString(parts[1]) + if err != nil { + return nil, fmt.Errorf("%w: decode payload: %v", ErrMalformed, err) + } + var claims map[string]any + if err := json.Unmarshal(pld, &claims); err != nil { + return nil, fmt.Errorf("%w: payload not JSON: %v", ErrMalformed, err) + } + return claims, nil +} + +// ParseDecryption parses the token and pulls out the strongly-typed +// decryption claims. Validates iss, aud, exp and that all required +// fields are present. +// +// expectedIssuer and expectedAudience cannot be empty. +func ParseDecryption(token, expectedIssuer, expectedAudience string) (*DecryptionClaims, error) { + claims, err := Parse(token) + if err != nil { + return nil, err + } + + getStr := func(k string) (string, error) { + v, ok := claims[k] + if !ok { + return "", fmt.Errorf("%w: %s", ErrMissingClaim, k) + } + s, ok := v.(string) + if !ok || s == "" { + return "", fmt.Errorf("%w: %s (not a non-empty string)", ErrMalformed, k) + } + return s, nil + } + + iss, err := getStr("iss") + if err != nil { + return nil, err + } + if iss != expectedIssuer { + return nil, fmt.Errorf("%w: got %q, want %q", ErrWrongIssuer, iss, expectedIssuer) + } + aud, err := getStr("aud") + if err != nil { + return nil, err + } + if aud != expectedAudience { + return nil, fmt.Errorf("%w: got %q, want %q", ErrWrongAudience, aud, expectedAudience) + } + + exp, ok := toUnix(claims["exp"]) + if !ok { + return nil, fmt.Errorf("%w: exp", ErrMissingClaim) + } + iat, _ := toUnix(claims["iat"]) + + expiresAt := time.Unix(exp, 0).UTC() + if time.Now().UTC().After(expiresAt) { + return nil, ErrExpired + } + + sub, err := getStr("sub") + if err != nil { + return nil, err + } + runID, err := getStr("run_id") + if err != nil { + return nil, err + } + companyID, err := getStr("company_id") + if err != nil { + return nil, err + } + dek, err := getStr("dek") + if err != nil { + return nil, err + } + alg, err := getStr("alg") + if err != nil { + return nil, err + } + fv, ok := toUnix(claims["format_version"]) + if !ok { + return nil, fmt.Errorf("%w: format_version", ErrMissingClaim) + } + + // sha256 is optional — empty is allowed (run had no recorded hash). + sha, _ := claims["sha256"].(string) + + return &DecryptionClaims{ + Issuer: iss, + Subject: sub, + Audience: aud, + IssuedAt: time.Unix(iat, 0).UTC(), + ExpiresAt: expiresAt, + RunID: runID, + CompanyID: companyID, + DEKBase64: dek, + Algorithm: alg, + FormatVersion: int(fv), + SHA256: sha, + }, nil +} + +func toUnix(v any) (int64, bool) { + switch n := v.(type) { + case float64: + return int64(n), true + case int64: + return n, true + case int: + return int64(n), true + case json.Number: + i, err := n.Int64() + return i, err == nil + } + return 0, false +} diff --git a/apps/backupy-decrypt/internal/jwt/jwt_test.go b/apps/backupy-decrypt/internal/jwt/jwt_test.go new file mode 100644 index 0000000..852b222 --- /dev/null +++ b/apps/backupy-decrypt/internal/jwt/jwt_test.go @@ -0,0 +1,94 @@ +package jwt + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// mintToken builds a JWT the same way the server does. The signature is +// intentionally not verified by the CLI; we still produce one so the +// token's structure is valid. +func mintToken(t *testing.T, claims map[string]any) string { + t.Helper() + hdr := map[string]string{"alg": "HS256", "typ": "JWT"} + hdrJSON, err := json.Marshal(hdr) + require.NoError(t, err) + pld, err := json.Marshal(claims) + require.NoError(t, err) + enc := base64.RawURLEncoding + signingInput := enc.EncodeToString(hdrJSON) + "." + enc.EncodeToString(pld) + mac := hmac.New(sha256.New, []byte("any-secret")) + mac.Write([]byte(signingInput)) + return signingInput + "." + enc.EncodeToString(mac.Sum(nil)) +} + +func validClaims() map[string]any { + return map[string]any{ + "iss": "backupy-server", + "sub": "user-123", + "aud": "backupy-decrypt", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Minute).Unix(), + "run_id": "run-1", + "company_id": "company-1", + "dek": base64.StdEncoding.EncodeToString(make([]byte, 32)), + "alg": "AES-256-GCM", + "format_version": 1, + "sha256": "abc", + } +} + +func TestParseDecryption_Success(t *testing.T) { + tok := mintToken(t, validClaims()) + c, err := ParseDecryption(tok, "backupy-server", "backupy-decrypt") + require.NoError(t, err) + require.Equal(t, "backupy-server", c.Issuer) + require.Equal(t, "user-123", c.Subject) + require.Equal(t, "run-1", c.RunID) + require.Equal(t, "company-1", c.CompanyID) + require.Equal(t, "AES-256-GCM", c.Algorithm) + require.Equal(t, 1, c.FormatVersion) +} + +func TestParseDecryption_Expired(t *testing.T) { + cl := validClaims() + cl["exp"] = time.Now().Add(-time.Minute).Unix() + tok := mintToken(t, cl) + _, err := ParseDecryption(tok, "backupy-server", "backupy-decrypt") + require.ErrorIs(t, err, ErrExpired) +} + +func TestParseDecryption_WrongIssuer(t *testing.T) { + cl := validClaims() + cl["iss"] = "evil" + tok := mintToken(t, cl) + _, err := ParseDecryption(tok, "backupy-server", "backupy-decrypt") + require.ErrorIs(t, err, ErrWrongIssuer) +} + +func TestParseDecryption_WrongAudience(t *testing.T) { + cl := validClaims() + cl["aud"] = "other" + tok := mintToken(t, cl) + _, err := ParseDecryption(tok, "backupy-server", "backupy-decrypt") + require.ErrorIs(t, err, ErrWrongAudience) +} + +func TestParseDecryption_MissingDEK(t *testing.T) { + cl := validClaims() + delete(cl, "dek") + tok := mintToken(t, cl) + _, err := ParseDecryption(tok, "backupy-server", "backupy-decrypt") + require.ErrorIs(t, err, ErrMissingClaim) +} + +func TestParseDecryption_Malformed(t *testing.T) { + _, err := ParseDecryption("not.a.jwt", "backupy-server", "backupy-decrypt") + require.ErrorIs(t, err, ErrMalformed) +} diff --git a/docs/03-agent-spec.md b/docs/03-agent-spec.md new file mode 100644 index 0000000..32dddfe --- /dev/null +++ b/docs/03-agent-spec.md @@ -0,0 +1,143 @@ +# 03. Agent — спецификация + +## Назначение + +Open-source Docker-сервис (MIT/Apache), который пользователь добавляет в свой `docker-compose.yml` рядом с приложением. Получает команды от сервера, делает бэкапы БД, шлёт в S3. + +## Запуск + +```yaml +services: + backup-agent: + image: backupservice/agent:latest + environment: + BACKUP_SERVER_URL: https://backupy.ru + BACKUP_AGENT_KEY: ${BACKUP_KEY} + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - backup-agent-state:/var/lib/backup-agent + restart: unless-stopped + +volumes: + backup-agent-state: +``` + +### Env-переменные (bootstrap) + +| Имя | Назначение | Required | +|---|---|---| +| `BACKUP_SERVER_URL` | Адрес control plane | да | +| `BACKUP_AGENT_KEY` | Ключ агента (секрет) | да | +| `BACKUP_LOG_LEVEL` | trace/debug/info/warn/error, default info | нет | +| `BACKUP_STATE_DIR` | Путь к state, default `/var/lib/backup-agent` | нет | + +Всё остальное (targets, schedules, S3 creds, retention, hooks) — приходит с сервера через `ConfigUpdate`. + +## Возможности + +### Auto-discovery БД через Docker socket +- Сканирует контейнеры по docker.sock (read-only). +- Распознаёт `postgres`, `mysql`, `mariadb`, `mongo`, `redis` по образам и `EXPOSE` портам. +- SQLite — обнаруживает файлы `.db`/`.sqlite` в смонтированных volume'ах подключённых контейнеров. +- Парсит env-переменные обнаруженных контейнеров (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `MYSQL_ROOT_PASSWORD`, etc.) — для предзаполнения форм в UI. +- Не передаёт plaintext-пароли на сервер. Отправляет только hint-список и имена переменных. +- Перепроверка discovery: раз в час + по событию docker events. + +### Persistent state в volume +- SQLite или BoltDB в `/var/lib/backup-agent/state.db`. +- Хранит: текущий config, очередь jobs, локальные логи, последний known config_version. +- Шифрование state опционально (key derived из BACKUP_AGENT_KEY). + +### WSS-канал +- Один long-lived connection на agent_id. +- Heartbeat каждые 30 сек. +- Reconnect: exponential backoff 1s → 2s → 4s → … → 60s (jitter ±20%). +- На разрыве — jobs не теряются, доделываются после reconnect (idempotent через run_id). + +### Backup pipeline +1. `pre_hooks` (опционально) — shell-команды до начала. +2. Dump БД через bundled tool. +3. Stream → zstd compression. +4. Stream → AES-256-GCM шифрование с DEK, полученным от сервера. +5. Stream → S3 multipart upload по presigned URL. +6. Smoke-validation: попытаться открыть только что загруженный файл, прочитать первые байты, валидировать заголовок (для pg_dump custom format — magic bytes `PGDMP`). +7. Расчёт SHA-256 целого файла (стримово, на лету). +8. `post_hooks` (опционально). +9. Отправка `BackupCompleted` со всей метой. + +### Поддерживаемые драйверы БД + +| Driver | Tool в образе | Стратегия | Phase | +|---|---|---|---| +| PostgreSQL | pg_dump (custom format) | Logical dump через TCP / unix socket | 1 | +| MySQL/MariaDB | mysqldump | Logical dump, `--single-transaction` | 1 | +| MongoDB | mongodump | Logical dump (archive format) | 2 | +| Redis | BGSAVE + copy RDB | Через `docker exec` или TCP `BGSAVE` + чтение dump.rdb | 2 | +| SQLite | `VACUUM INTO` или копия с rsync | File-based | 2 | + +### Health checks (Phase 2) +- HTTP/HTTPS: GET URL, проверка status code и опционально body match. +- TCP: connect-then-close на host:port. +- Custom interval per check (default 60s). +- Timeout per check (default 10s). +- Результат сразу шлётся серверу. + +### Pre/Post hooks (Phase 2, opt-in) +- Shell-команды, выполняемые в контексте контейнера агента. +- Опции: `docker exec <container> <cmd>` или прямой shell. +- Timeout per hook (default 30s). +- В UI явный warning: «Это выполнит код на вашем хосте. Используйте только команды, которым доверяете». + +### Auto-update (Phase 2) +- Сервер шлёт `SelfUpdate{target_version, binary_url, sha256, cosign_signature}`. +- Агент валидирует cosign-подпись (публичный ключ зашит в бинарь). +- Скачивает в `/var/lib/backup-agent/bin/<version>`. +- Graceful binary swap: запускает новый бинарь, передаёт открытые сокеты через fd-passing, новый агент шлёт `HealthyAfterUpdate`, старый exit'ит. +- Откат: если новый не пришёл healthy за 60 сек — старый продолжает работать, шлёт alert. +- Opt-out через config (`auto_update_enabled=false`). + +## Требования к образу + +| Параметр | Значение | +|---|---| +| Базовый образ | distroless или alpine | +| Размер | < 50 MB | +| Архитектуры | linux/amd64, linux/arm64 | +| Pinned binaries | pg_dump, mysqldump, mongodump, redis-cli — версии явные | +| User | non-root (uid 1000) | +| Healthcheck | `HEALTHCHECK CMD agent health-check` | +| Tags | `1.x.y`, `1.x`, `1`, `latest` | +| Подпись | cosign sign-blob | + +## Безопасность агента + +- TLS 1.3 ко всем endpoint'ам. +- Pinning публичного ключа сервера (зашит в бинарь). +- Docker socket монтируется read-only. +- `BACKUP_AGENT_KEY` никогда не пишется в логи. +- Локальный state шифруется (опционально включается). +- Healthcheck endpoint (если будет) — только на localhost. +- Capabilities контейнера: drop ALL. + +## Метрики (Prometheus, localhost) + +``` +backup_agent_up +backup_agent_config_version +backup_agent_jobs_total{status="success|failed"} +backup_agent_backup_size_bytes +backup_agent_backup_duration_seconds +backup_agent_queue_depth +backup_agent_reconnects_total +backup_agent_health_check_duration_seconds{check_id="..."} +``` + +## CLI агента (внутренние команды, не для пользователя) + +```bash +agent run # default, запускает service loop +agent version # вывести версию +agent health-check # для Docker HEALTHCHECK +agent dump-state # debug: вывести state в stdout +agent self-update <ver> # ручной триггер обновления +``` diff --git a/docs/07-api-contract.md b/docs/07-api-contract.md new file mode 100644 index 0000000..a426dc7 --- /dev/null +++ b/docs/07-api-contract.md @@ -0,0 +1,635 @@ +# 07. API-контракты + +## 1. Транспорт Agent ↔ Server + +- **Протокол**: WebSocket Secure (WSS) на :443, бинарные фреймы с Protobuf-payload'ами. +- **Альтернатива** для Phase 3+: gRPC bidirectional stream. +- Длинноживущее соединение, инициируемое агентом. +- Heartbeat: PING каждые 30 секунд, RTO 90 секунд. +- Reconnect: exponential backoff 1s → 2s → 4s → … → 60s cap, jitter ±20%. + +## 2. Аутентификация WSS + +1. Агент открывает WSS-соединение с заголовком `Authorization: Bearer <agent_key>`. +2. Сервер валидирует key (argon2-хэш в БД), достаёт agent_id. +3. Первое сообщение от агента — `Register`, сервер отвечает `RegisterAck` с config snapshot и assigned `session_id`. +4. Все последующие сообщения содержат monotonic `seq` для дедупликации. + +## 3. Envelope + +```protobuf +syntax = "proto3"; +package backup.v1; + +message Envelope { + uint64 seq = 1; // monotonic, per-direction + uint64 ts_ms = 2; // unix ms, отправителя + string correlation_id = 3; // для request/response пар + oneof payload { + // ----- agent → server ----- + Register register = 10; + Heartbeat heartbeat = 11; + DiscoveryReport discovery = 12; + JobUpdate job_update = 13; + BackupCompleted backup_completed = 14; + HealthCheckResult health_result = 15; + LogEvent log = 16; + RestoreUpdate restore_update = 17; + Ack ack = 18; + + // ----- server → agent ----- + RegisterAck register_ack = 50; + ConfigUpdate config_update = 51; + RunBackup run_backup = 52; + CancelJob cancel_job = 53; + RunHealthCheck run_health_check = 54; + SelfUpdate self_update = 56; + Ping ping = 57; + } +} +``` + +> **Изменение относительно v1:** `RunRestore` удалён из MVP. В Phase 1 нет restore-в-БД, только download через REST endpoint (`GET /runs/:id/download-url` + локальная расшифровка CLI-утилитой). См. [04-server-spec.md → Download](./04-server-spec.md). + +## 4. Сообщения Agent → Server + +### Register + +```protobuf +message Register { + string agent_version = 1; + string hostname = 2; + string os = 3; // "linux" + string arch = 4; // "amd64" + string docker_version = 5; + repeated string capabilities = 6; // ["pg_dump", "mysqldump", "docker_discovery"] + uint64 last_known_config_version = 7; +} +``` + +### Heartbeat + +```protobuf +message Heartbeat { + uint64 config_version = 1; + AgentMetrics metrics = 2; + repeated string active_job_ids = 3; +} +message AgentMetrics { + float cpu_percent = 1; + uint64 mem_used_bytes = 2; + uint64 mem_total_bytes = 3; + uint64 disk_used_bytes = 4; + uint64 disk_total_bytes = 5; + uint32 queue_depth = 6; +} +``` + +Частота: каждые 30 секунд. + +### DiscoveryReport + +```protobuf +message DiscoveryReport { + repeated DiscoveredContainer containers = 1; +} +message DiscoveredContainer { + string container_id = 1; + string name = 2; + string image = 3; // "postgres:16" + string detected_db_type = 4; // "postgresql" + repeated string networks = 5; + map<string,string> env_hints = 6; // отфильтрованные env без секретов + repeated PortBinding ports = 7; +} +message PortBinding { + uint32 container_port = 1; + uint32 host_port = 2; + string protocol = 3; +} +``` + +Отправляется при старте и при изменениях docker events. + +### JobUpdate + +```protobuf +message JobUpdate { + string job_id = 1; + JobStatus status = 2; + uint32 progress_percent = 3; + string current_step = 4; // "dumping", "compressing", "uploading" + string error_message = 5; // если status=FAILED +} +enum JobStatus { + JOB_STATUS_UNSPECIFIED = 0; + QUEUED = 1; + RUNNING = 2; + SUCCESS = 3; + FAILED = 4; + CANCELLED = 5; +} +``` + +### BackupCompleted + +```protobuf +message BackupCompleted { + string job_id = 1; + string run_id = 2; + string s3_key = 3; + uint64 size_bytes = 4; + string sha256 = 5; + uint64 duration_ms = 6; + string dek_kms_id = 7; + bytes encrypted_dek = 8; + string compression = 9; // "zstd" + string db_engine_version = 10; // "PostgreSQL 16.2" +} +``` + +### HealthCheckResult + +```protobuf +message HealthCheckResult { + string check_id = 1; + uint64 ts_ms = 2; + bool ok = 3; + uint32 latency_ms = 4; + string error = 5; + uint32 status_code = 6; // для HTTP +} +``` + +### LogEvent + +```protobuf +message LogEvent { + uint64 ts_ms = 1; + LogLevel level = 2; + string job_id = 3; // опционально + string message = 4; + map<string,string> fields = 5; +} +enum LogLevel { TRACE=0; DEBUG=1; INFO=2; WARN=3; ERROR=4; } +``` + +### RestoreUpdate + +```protobuf +message RestoreUpdate { + string restore_id = 1; + JobStatus status = 2; + uint32 progress_percent = 3; + string current_step = 4; + string error_message = 5; +} +``` + +### Ack + +```protobuf +message Ack { + string correlation_id = 1; + bool accepted = 2; + string reason = 3; // если accepted=false +} +``` + +## 5. Сообщения Server → Agent + +### RegisterAck + +```protobuf +message RegisterAck { + string session_id = 1; + AgentConfig config = 2; + uint32 heartbeat_interval_sec = 3; + string server_time_iso = 4; +} +``` + +### AgentConfig (используется в RegisterAck и ConfigUpdate) + +```protobuf +message AgentConfig { + uint64 version = 1; + repeated Target targets = 2; + repeated BackupJobSpec jobs = 3; + repeated HealthCheckSpec health_checks = 4; + MaintenanceWindow maintenance = 5; + AdvancedSettings advanced = 6; +} + +message Target { + string id = 1; + DbType type = 2; + string display_name = 3; + ConnectionConfig connection = 4; +} + +enum DbType { + DB_UNSPECIFIED = 0; + POSTGRESQL = 1; + MYSQL = 2; + MARIADB = 3; + MONGODB = 4; + REDIS = 5; + SQLITE = 6; +} + +message ConnectionConfig { + string host = 1; + uint32 port = 2; + string database = 3; + string username = 4; + string password_secret_ref = 5; // ссылка на зашифрованный секрет + string container_id = 6; // для docker exec strategy + ConnectionStrategy strategy = 7; +} +enum ConnectionStrategy { TCP = 0; DOCKER_EXEC = 1; UNIX_SOCKET = 2; } + +message BackupJobSpec { + string id = 1; + string target_id = 2; + string cron = 3; // UTC + uint32 retention_days = 4; + string compression = 5; // "zstd" | "gzip" | "none" + bool encryption_enabled = 6; + repeated string pre_hooks = 7; + repeated string post_hooks = 8; + RetryPolicy retry = 9; + uint32 timeout_sec = 10; +} + +message RetryPolicy { + uint32 max_attempts = 1; + repeated uint32 backoff_seconds = 2; // [60, 300, 1800] +} + +message HealthCheckSpec { + string id = 1; + string name = 2; + CheckType type = 3; + string target = 4; // URL или host:port + uint32 interval_sec = 5; + uint32 timeout_sec = 6; + uint32 expected_status = 7; // для HTTP +} +enum CheckType { HTTP = 0; HTTPS = 1; TCP = 2; } + +message MaintenanceWindow { + string start_utc = 1; // "02:00" + string end_utc = 2; // "05:00" +} + +message AdvancedSettings { + uint32 max_parallel_jobs = 1; + string log_level = 2; + bool auto_update_enabled = 3; +} +``` + +### RunBackup + +```protobuf +message RunBackup { + string job_id = 1; + string run_id = 2; + bool manual_trigger = 3; + S3UploadCreds upload_creds = 4; + bytes encrypted_dek = 5; // DEK для шифрования, обёрнутый KMS +} +message S3UploadCreds { + string presigned_put_url = 1; + uint64 expires_at_ms = 2; + string final_s3_key = 3; +} +``` + +### CancelJob, RunHealthCheck + +```protobuf +message CancelJob { string job_id = 1; } +message RunHealthCheck { string check_id = 1; } +``` + +### ~~RunRestore~~ — удалено в MVP + +В Phase 1 функция «получить данные обратно» работает через REST API (см. ниже эндпоинты `/runs/:id/download-url` и `/runs/:id/decryption-token`) + локальную CLI-утилиту `backupy-decrypt`. Агент в этом потоке не участвует. + +Restore-в-БД через агент (RESTORE_MODE: NEW_DATABASE / ANOTHER_AGENT) — Phase 3. К тому моменту в proto будет добавлен новый message `RunRestore` (proto v2). + +### SelfUpdate + +```protobuf +message SelfUpdate { + string target_version = 1; + string binary_url = 2; + string sha256 = 3; + bytes cosign_signature = 4; + bool force = 5; +} +``` + +--- + +## 6. REST API + +База: `https://api.backupy.ru/v1`. +Auth: `Authorization: Bearer <token>` (session-token из UI или API-key). + +### Auth & Account + +| Метод | Путь | Назначение | +|---|---|---| +| POST | `/auth/signup` | email+password регистрация | +| POST | `/auth/login` | email+password | +| POST | `/auth/magic-link/request` | запрос magic link | +| GET | `/auth/magic-link/verify?token=` | подтверждение | +| GET | `/auth/oauth/github` | OAuth redirect | +| GET | `/auth/oauth/github/callback` | OAuth callback | +| GET | `/auth/oauth/google` | OAuth redirect | +| GET | `/auth/oauth/google/callback` | OAuth callback | +| POST | `/auth/logout` | | +| POST | `/auth/password-reset/request` | | +| POST | `/auth/password-reset/confirm` | | +| GET | `/me` | текущий пользователь | +| PATCH | `/me` | обновить профиль | +| DELETE | `/me` | account deletion (start grace period) | +| GET | `/me/export` | data export JSON | +| POST | `/me/2fa/enable` | включить TOTP | +| POST | `/me/2fa/disable` | | + +### Agents + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/agents` | список агентов | +| POST | `/agents` | создать (возвращает раз secret key) | +| GET | `/agents/:id` | детали | +| PATCH | `/agents/:id` | изменить имя/настройки | +| DELETE | `/agents/:id` | | +| POST | `/agents/:id/rotate-key` | новый ключ | +| GET | `/agents/:id/config` | текущий config (видимый пользователю) | +| PATCH | `/agents/:id/config` | изменить (создаёт новую version) | +| GET | `/agents/:id/discovered` | то, что обнаружил Docker socket | +| GET | `/agents/:id/metrics` | последние метрики | + +### Targets & Jobs + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/agents/:id/targets` | список target'ов | +| POST | `/agents/:id/targets` | создать target | +| PATCH | `/targets/:id` | | +| DELETE | `/targets/:id` | | +| POST | `/targets/:id/test-connection` | проверка коннекта | +| GET | `/agents/:id/jobs` | jobs | +| POST | `/agents/:id/jobs` | новый job | +| GET | `/jobs/:id` | детали | +| PATCH | `/jobs/:id` | редактирование | +| DELETE | `/jobs/:id` | | +| POST | `/jobs/:id/run` | manual run, `Idempotency-Key` поддерживается | + +### Backup Runs + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/runs` | каталог с фильтрами `?agent_id=&target_id=&from=&to=&status=` | +| GET | `/runs/:id` | детали | +| GET | `/runs/:id/download-url` | presigned GET URL для скачивания зашифрованного объекта (TTL 15 min) | +| POST | `/runs/:id/decryption-token` | временный JWT с plaintext DEK для CLI-утилиты (TTL 15 min) | +| DELETE | `/runs/:id` | удалить бэкап вручную | + +> Restore-в-БД (`POST /runs/:id/restore` + `/restores/:id`) — **Phase 3**, не реализуется в MVP. + +### Health Checks + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/health-checks` | | +| POST | `/health-checks` | | +| PATCH | `/health-checks/:id` | | +| DELETE | `/health-checks/:id` | | +| GET | `/health-checks/:id/results` | history с временным фильтром | +| GET | `/health-checks/:id/uptime?period=24h\|7d\|30d\|90d` | aggregated | + +### Storage Profiles + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/storage-profiles` | managed + BYO | +| POST | `/storage-profiles` | добавить BYO-S3 | +| PATCH | `/storage-profiles/:id` | | +| DELETE | `/storage-profiles/:id` | | +| POST | `/storage-profiles/:id/test` | тестовое PUT/GET | +| GET | `/storage-profiles/usage` | сколько занято | + +### Status Page + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/status-page` | конфиг страницы | +| PATCH | `/status-page` | | +| POST | `/status-page/verify-dns` | проверка CNAME | +| POST | `/status-page/incidents` | создать incident | +| PATCH | `/incidents/:id` | обновить status (Investigating → Resolved) | +| GET | `/incidents` | список | +| GET | `/status-page/subscribers` | | + +### Public (без auth) + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/public/status/:subdomain` | публичная страница | +| GET | `/public/status/:subdomain/rss` | RSS feed | +| POST | `/public/status/:subdomain/subscribe` | подписка | +| GET | `/public/status/:subdomain/unsubscribe?token=` | | + +### Notifications + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/notification-channels` | | +| POST | `/notification-channels` | email/telegram/webhook | +| PATCH | `/notification-channels/:id` | | +| DELETE | `/notification-channels/:id` | | +| POST | `/notification-channels/:id/test` | отправить тестовое | +| GET | `/notification-preferences` | | +| PATCH | `/notification-preferences` | | + +### Audit Log & API Tokens + +| Метод | Путь | Назначение | +|---|---|---| +| GET | `/audit-log` | с фильтрами | +| GET | `/audit-log/export` | CSV | +| GET | `/api-tokens` | список (без plaintext) | +| POST | `/api-tokens` | создать | +| DELETE | `/api-tokens/:id` | | + +--- + +## 6a. Admin REST API (`/admin/v1/`) + +Базовый путь: `https://api.backupy.ru/admin/v1`. +Auth: тот же Bearer token, но проверяется `role=admin`. Не-admin запрос → `403 PERMISSION_DENIED`. + +### Companies (admin only) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/companies` | список с фильтрами `?plan=&status=&country=&search=` | 1 | +| POST | `/companies` | создать вручную (для enterprise-онбординга) | 1 | +| GET | `/companies/:id` | детальная карточка (users, agents count, payment history placeholder, internal notes) | 1 | +| PATCH | `/companies/:id` | обновить (имя, контактный email, страна, plan, status) | 1 | +| POST | `/companies/:id/suspend` | приостановить — все агенты компании перестают получать команды | 1 | +| POST | `/companies/:id/reactivate` | вернуть в active | 1 | +| POST | `/companies/:id/impersonate` | создать временную user-session под owner-юзером компании | 1 | +| GET | `/companies/:id/notes` | internal notes (admin-only) | 1 | +| POST | `/companies/:id/notes` | добавить заметку | 1 | +| POST | `/companies/:id/change-plan` | сменить план (без биллинга — биллинг Phase 3) | 2 | + +### Users (admin only) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/users` | все юзеры с фильтрами `?company_id=&role=&status=&search=` | 1 | +| POST | `/users` | создать юзера в конкретной компании (или с auto-создаваемой) | 1 | +| GET | `/users/:id` | детали юзера | 1 | +| POST | `/users/:id/suspend` | заблокировать sign-in и API | 1 | +| POST | `/users/:id/reactivate` | | 1 | +| POST | `/users/:id/impersonate` | временная user-session | 1 | +| POST | `/users/:id/reset-password` | отправить reset-email | 1 | +| DELETE | `/users/:id` | начать GDPR-удаление (полный flow — Phase 3) | 1 (start) / 3 (full) | + +### Plans (admin only) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/plans` | все тарифы (active + inactive) | 1 | +| POST | `/plans` | создать новый | 2 | +| PATCH | `/plans/:id` | обновить (цена, лимиты, фичи, active) | 2 | +| DELETE | `/plans/:id` | удалить (если customers=0; иначе ошибка) | 2 | +| GET | `/plans/distribution` | сколько customers на каждом плане + MRR | 2 | + +### Infrastructure (admin only, read-only) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/infra/s3` | список наших S3 bucket'ов: размер, объекты, стоимость | 1 | +| GET | `/infra/kms` | KMS keys active, operations today, cost | 1 | +| GET | `/infra/cluster` | K8s cluster CPU/memory | 1 | +| GET | `/infra/db` | PostgreSQL size, replication lag, connections | 1 | +| GET | `/infra/redis` | Redis memory usage | 1 | +| GET | `/infra/cost-breakdown` | детализация месячных расходов + gross margin расчёт | 2 | + +### Admin audit log + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/admin-audit` | список admin-actions с фильтрами | 1 | +| GET | `/admin-audit/export` | CSV | 2 | + +### System settings (admin only) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/system-settings` | branding, signup config, feature flags | 2 | +| PATCH | `/system-settings` | обновить | 2 | +| GET | `/system-settings/feature-flags` | список фич с per-company overrides | 2 | +| POST | `/system-settings/feature-flags/:flag/enable` | включить глобально | 2 | +| POST | `/system-settings/feature-flags/:flag/disable` | выключить глобально | 2 | + +### Admin team + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/admin-team` | список админов с ролями | 2 | +| POST | `/admin-team/invite` | пригласить нового админа | 2 | +| PATCH | `/admin-team/:id` | сменить роль (super_admin / support / billing) | 2 | +| DELETE | `/admin-team/:id` | удалить админа | 2 | + +### Payments & Revenue (Phase 3, заглушки в Phase 2) + +| Метод | Путь | Назначение | Phase | +|---|---|---|---| +| GET | `/payments` | все транзакции с фильтрами | 3 | +| POST | `/payments/:id/refund` | refund-action | 3 | +| POST | `/payments/:id/retry` | повторить failed charge | 3 | +| GET | `/analytics/revenue` | MRR, ARR, churn, conversion, cohorts | 3 | + +--- + +## 7. Модель ошибок + +```json +{ + "error": { + "code": "RESOURCE_NOT_FOUND", + "message": "Agent not found", + "details": { "agent_id": "agt_..." }, + "trace_id": "01HM..." + } +} +``` + +Коды: +- `AUTH_FAILED` +- `PERMISSION_DENIED` +- `RESOURCE_NOT_FOUND` +- `VALIDATION_ERROR` +- `RATE_LIMITED` +- `CONFLICT` +- `PRECONDITION_FAILED` +- `INTERNAL` +- `SERVICE_UNAVAILABLE` + +## 8. Идемпотентность + +- `POST /jobs/:id/run` и `POST /runs/:id/restore` принимают `Idempotency-Key` header. +- Сервер кеширует ответ по ключу на 24 часа. +- `BackupCompleted` от агента дедуплицируется по `run_id` на сервере (upsert). + +## 9. Pagination + +- Query params: `?limit=50&cursor=<opaque>`. +- Response: +```json +{ + "items": [...], + "next_cursor": "...", + "has_more": true +} +``` +- Limit max: 100. + +## 10. Real-time (для UI) + +- SSE endpoint: `GET /events?topics=jobs,agents,restores` (auth required). +- События: + - `agent.connected`, `agent.disconnected`, `agent.config.applied` + - `job.started`, `job.progress`, `job.completed`, `job.failed` + - `restore.started`, `restore.progress`, `restore.completed`, `restore.failed` + - `incident.opened`, `incident.resolved` + +## 11. Версионирование + +- API: `/v1/...`. Breaking changes → `/v2/`. +- Proto: семантическое версионирование пакета (`backup.v1`, `backup.v2`). +- Агент шлёт `agent_version`, сервер договаривается о минимальной поддерживаемой версии. +- Минимум 6 месяцев backward compatibility между proto-версиями. + +## 12. Rate limits (defaults) + +| Endpoint | Limit | +|---|---| +| `/auth/login` | 5/min/IP | +| `/auth/signup` | 3/hour/IP | +| `/auth/magic-link/request` | 3/hour/email | +| `/auth/password-reset/request` | 3/hour/email | +| API (auth required) | 60/min/token | +| `/public/status/*` | 60/min/IP | + +Превышение → 429 с `Retry-After`. diff --git a/packages/proto/.gitignore b/packages/proto/.gitignore new file mode 100644 index 0000000..3db5532 --- /dev/null +++ b/packages/proto/.gitignore @@ -0,0 +1,7 @@ +# Backupy protobuf module +# +# Generated Go code is committed (small Go project, easier debugging, no +# need for a buf install in every consumer). Ignore only local buf caches +# and tool downloads. + +.buf-cache/ diff --git a/packages/proto/README.md b/packages/proto/README.md new file mode 100644 index 0000000..1a15b4a --- /dev/null +++ b/packages/proto/README.md @@ -0,0 +1,91 @@ +# `packages/proto` + +Canonical Protobuf wire format for Backupy. + +Both the Go agent (`apps/agent`) and the Go server (`apps/server`) consume the +generated bindings produced from these `.proto` files. The contract itself is +described in [`docs/07-api-contract.md`](../../docs/07-api-contract.md) — this +package is the executable form of that document. + +## Layout + +``` +packages/proto/ +├── buf.yaml # module config: lint (STANDARD), breaking (FILE) +├── buf.gen.yaml # codegen plugin config (protoc-gen-go) +├── v1/ +│ ├── common.proto # shared enums + messages +│ ├── envelope.proto # Envelope wrapper (oneof of all payloads) +│ ├── agent_to_server.proto # Register, Heartbeat, JobUpdate, ... +│ └── server_to_agent.proto # RegisterAck, ConfigUpdate, RunBackup, ... +└── gen/ + └── go/v1/ # generated Go bindings (committed) +``` + +Package name: `backup.v1`. +Go import path: `github.com/backupy/backupy/packages/proto/gen/go/v1` +(`backupv1` short name). + +## Prerequisites + +Install [buf](https://buf.build/docs/installation) and the Go plugin once +per workstation: + +```sh +# from the repo root +make tools +``` + +That installs `buf` and `protoc-gen-go` into `$GOBIN` (or `$GOPATH/bin`). + +## Regenerating code + +From the repository root: + +```sh +make gen +``` + +This `cd`s into `packages/proto` and runs `buf generate`. The output is +written to `packages/proto/gen/go/v1/`. + +Alternatively, from this directory: + +```sh +buf generate +``` + +## Linting and breaking-change detection + +```sh +# from the repo root +make lint # includes `buf lint` + +# or directly +cd packages/proto +buf lint +buf breaking --against '.git#branch=main,subdir=packages/proto' +``` + +`buf.yaml` enables the `STANDARD` lint set and `FILE`-level breaking-change +detection. Field renames or tag-number changes in any committed `.proto` +will fail CI. + +## Adding a new message + +1. Pick the right file (`agent_to_server.proto` vs `server_to_agent.proto`), + or add a new shared type to `common.proto`. +2. Use a fresh field number — never reuse a `reserved` tag. +3. If it is a new top-level message that flows over WSS, also add it to the + `Envelope.payload` `oneof` in `envelope.proto`, picking a stable tag in + the matching direction's range (agent→server: 10..49, server→agent: + 50..99). +4. Run `make gen` and commit both the `.proto` change and the regenerated + Go files in the same commit. +5. Run `buf breaking` locally before pushing. + +## Versioning + +The package is `backup.v1`. Any breaking change requires a new package +(`backup.v2`) plus at least six months of dual-support — see +[`docs/07-api-contract.md` §11](../../docs/07-api-contract.md). diff --git a/packages/proto/backupv1/agent_to_server.proto b/packages/proto/backupv1/agent_to_server.proto new file mode 100644 index 0000000..19517d9 --- /dev/null +++ b/packages/proto/backupv1/agent_to_server.proto @@ -0,0 +1,151 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Messages flowing from the agent to the server over the long-lived WSS +// connection. Each is wrapped in an Envelope (see envelope.proto). + +syntax = "proto3"; + +package backup.v1; + +import "backupv1/common.proto"; + +option go_package = "github.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1"; + +// ----------------------------------------------------------------------------- +// Register / Heartbeat +// ----------------------------------------------------------------------------- + +// Register is the first message the agent sends after authenticating the +// WSS connection. The server replies with RegisterAck. +// See docs/07-api-contract.md §4 (Register). +message Register { + string agent_version = 1; + string hostname = 2; + string os = 3; // "linux" + string arch = 4; // "amd64" + string docker_version = 5; + repeated string capabilities = 6; // ["pg_dump", "mysqldump", "docker_discovery"] + uint64 last_known_config_version = 7; +} + +// Heartbeat is sent every heartbeat_interval_sec (default 30s). +// See docs/07-api-contract.md §4 (Heartbeat). +message Heartbeat { + uint64 config_version = 1; + AgentMetrics metrics = 2; + repeated string active_job_ids = 3; +} + +// ----------------------------------------------------------------------------- +// Discovery +// ----------------------------------------------------------------------------- + +// DiscoveryReport is sent on startup and whenever docker events change the set +// of running containers. See docs/07-api-contract.md §4 (DiscoveryReport). +message DiscoveryReport { + repeated DiscoveredContainer containers = 1; +} + +message DiscoveredContainer { + string container_id = 1; + string name = 2; + string image = 3; // "postgres:16" + string detected_db_type = 4; // "postgresql" + repeated string networks = 5; + map<string, string> env_hints = 6; // env vars filtered to remove secrets + repeated PortBinding ports = 7; +} + +// ----------------------------------------------------------------------------- +// Job lifecycle +// ----------------------------------------------------------------------------- + +// JobUpdate is a progress tick for a running backup job. +// See docs/07-api-contract.md §4 (JobUpdate). +// +// run_id was appended (field 6) after the initial Phase-1 wire so the +// scheduler can correlate a tick to a specific BackupRun row. Older +// agents that omit it are tolerated as "job-level" updates (no run row +// transition). +message JobUpdate { + string job_id = 1; + JobStatus status = 2; + uint32 progress_percent = 3; + string current_step = 4; // "dumping", "compressing", "uploading" + string error_message = 5; // populated when status == FAILED + string run_id = 6; // correlates to BackupRun.id; appended in v1.1 +} + +// BackupCompleted is sent once after a successful S3 upload, carrying the +// metadata the server needs to persist a backup_runs row. +// See docs/07-api-contract.md §4 (BackupCompleted). +message BackupCompleted { + string job_id = 1; + string run_id = 2; + string s3_key = 3; + uint64 size_bytes = 4; + string sha256 = 5; + uint64 duration_ms = 6; + string dek_kms_id = 7; + bytes encrypted_dek = 8; + string compression = 9; // "zstd" + string db_engine_version = 10; // "PostgreSQL 16.2" +} + +// ----------------------------------------------------------------------------- +// Health checks +// ----------------------------------------------------------------------------- + +// HealthCheckResult is the outcome of a single probe run. +// See docs/07-api-contract.md §4 (HealthCheckResult). +message HealthCheckResult { + string check_id = 1; + uint64 ts_ms = 2; + bool ok = 3; + uint32 latency_ms = 4; + string error = 5; + uint32 status_code = 6; // populated for HTTP/HTTPS probes +} + +// ----------------------------------------------------------------------------- +// Logs +// ----------------------------------------------------------------------------- + +// LogEvent is a structured log line streamed from the agent. +// See docs/07-api-contract.md §4 (LogEvent). +message LogEvent { + uint64 ts_ms = 1; + LogLevel level = 2; + string job_id = 3; // optional + string message = 4; + map<string, string> fields = 5; +} + +// ----------------------------------------------------------------------------- +// Restore (download-only stub in MVP) +// ----------------------------------------------------------------------------- + +// RestoreUpdate is reserved for future agent-driven restore flows. In MVP the +// agent never receives a RunRestore command — restore happens through the REST +// download endpoint and the local backupy-decrypt CLI. Kept in the contract so +// the field number is stable for Phase 3. +// See docs/07-api-contract.md §4 (RestoreUpdate). +message RestoreUpdate { + string restore_id = 1; + JobStatus status = 2; + uint32 progress_percent = 3; + string current_step = 4; + string error_message = 5; +} + +// ----------------------------------------------------------------------------- +// Generic acknowledgement +// ----------------------------------------------------------------------------- + +// Ack responds to a server message identified by correlation_id. +// See docs/07-api-contract.md §4 (Ack). +message Ack { + string correlation_id = 1; + bool accepted = 2; + string reason = 3; // populated when accepted == false +} diff --git a/packages/proto/backupv1/common.proto b/packages/proto/backupv1/common.proto new file mode 100644 index 0000000..53cb311 --- /dev/null +++ b/packages/proto/backupv1/common.proto @@ -0,0 +1,125 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Shared enums and messages used across agent_to_server and server_to_agent +// payloads. Keep this file dependency-free — only primitives and other +// definitions from this same file may appear here. + +syntax = "proto3"; + +package backup.v1; + +option go_package = "github.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1"; + +// ----------------------------------------------------------------------------- +// Enums +// ----------------------------------------------------------------------------- + +// JobStatus is the lifecycle state for a backup or restore job. +// See docs/07-api-contract.md §4 (JobUpdate, RestoreUpdate). +enum JobStatus { + JOB_STATUS_UNSPECIFIED = 0; + QUEUED = 1; + RUNNING = 2; + SUCCESS = 3; + FAILED = 4; + CANCELLED = 5; +} + +// DbType enumerates supported database engines for targets. +// See docs/07-api-contract.md §5 (Target). +enum DbType { + DB_UNSPECIFIED = 0; + POSTGRESQL = 1; + MYSQL = 2; + MARIADB = 3; + MONGODB = 4; + REDIS = 5; + SQLITE = 6; +} + +// ConnectionStrategy is how the agent reaches the database. +// See docs/07-api-contract.md §5 (ConnectionConfig). +enum ConnectionStrategy { + TCP = 0; + DOCKER_EXEC = 1; + UNIX_SOCKET = 2; +} + +// CheckType enumerates supported health-check probe types. +// See docs/07-api-contract.md §5 (HealthCheckSpec). +enum CheckType { + HTTP = 0; + HTTPS = 1; + TCP_CHECK = 2; +} + +// LogLevel is the severity for LogEvent records streamed by the agent. +// See docs/07-api-contract.md §4 (LogEvent). +enum LogLevel { + TRACE = 0; + DEBUG = 1; + INFO = 2; + WARN = 3; + ERROR = 4; +} + +// ----------------------------------------------------------------------------- +// Shared messages +// ----------------------------------------------------------------------------- + +// AgentMetrics is the snapshot of host resource usage attached to Heartbeat. +// See docs/07-api-contract.md §4 (Heartbeat). +message AgentMetrics { + float cpu_percent = 1; + uint64 mem_used_bytes = 2; + uint64 mem_total_bytes = 3; + uint64 disk_used_bytes = 4; + uint64 disk_total_bytes = 5; + uint32 queue_depth = 6; +} + +// PortBinding describes a single container port published on the host. +// See docs/07-api-contract.md §4 (DiscoveryReport). +message PortBinding { + uint32 container_port = 1; + uint32 host_port = 2; + string protocol = 3; +} + +// S3UploadCreds is a presigned PUT URL the agent uses to upload the encrypted +// backup blob. See docs/07-api-contract.md §5 (RunBackup). +message S3UploadCreds { + string presigned_put_url = 1; + uint64 expires_at_ms = 2; + string final_s3_key = 3; +} + +// S3DownloadCreds is a presigned GET URL for fetching a backup blob (reserved +// for future restore-via-agent flows). Not used in MVP — Phase 3. +message S3DownloadCreds { + string presigned_get_url = 1; + uint64 expires_at_ms = 2; + string s3_key = 3; +} + +// RetryPolicy controls how the agent retries a failed backup job. +// See docs/07-api-contract.md §5 (BackupJobSpec). +message RetryPolicy { + uint32 max_attempts = 1; + repeated uint32 backoff_seconds = 2; // e.g. [60, 300, 1800] +} + +// MaintenanceWindow defines a UTC time-of-day window where backups are paused. +// See docs/07-api-contract.md §5 (AgentConfig). +message MaintenanceWindow { + string start_utc = 1; // "02:00" + string end_utc = 2; // "05:00" +} + +// AdvancedSettings carries miscellaneous tunables pushed in AgentConfig. +// See docs/07-api-contract.md §5 (AgentConfig). +message AdvancedSettings { + uint32 max_parallel_jobs = 1; + string log_level = 2; + bool auto_update_enabled = 3; +} diff --git a/packages/proto/backupv1/envelope.proto b/packages/proto/backupv1/envelope.proto new file mode 100644 index 0000000..b9f1f2d --- /dev/null +++ b/packages/proto/backupv1/envelope.proto @@ -0,0 +1,49 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Envelope is the single top-level wrapper carried in every WSS binary frame +// in either direction. The `payload` oneof selects which concrete message is +// being transported; field numbers must remain stable forever. + +syntax = "proto3"; + +package backup.v1; + +import "backupv1/agent_to_server.proto"; +import "backupv1/server_to_agent.proto"; + +option go_package = "github.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1"; + +// Envelope wraps every WSS message in both directions. +// See docs/07-api-contract.md §3. +message Envelope { + uint64 seq = 1; // monotonic, per-direction + uint64 ts_ms = 2; // unix milliseconds, set by sender + string correlation_id = 3; // links request/response pairs (e.g. Ping/Ack) + + oneof payload { + // ---------------- agent -> server ---------------- + Register register = 10; + Heartbeat heartbeat = 11; + DiscoveryReport discovery = 12; + JobUpdate job_update = 13; + BackupCompleted backup_completed = 14; + HealthCheckResult health_result = 15; + LogEvent log = 16; + RestoreUpdate restore_update = 17; + Ack ack = 18; + + // ---------------- server -> agent ---------------- + RegisterAck register_ack = 50; + ConfigUpdate config_update = 51; + RunBackup run_backup = 52; + CancelJob cancel_job = 53; + RunHealthCheck run_health_check = 54; + // 55 reserved: previously RunRestore (removed in MVP, see docs/07 §5). + SelfUpdate self_update = 56; + Ping ping = 57; + } + + // Reserve the tag previously occupied by RunRestore so the wire format + // remains compatible if a future v2 reintroduces it under a new message. + reserved 55; +} diff --git a/packages/proto/backupv1/server_to_agent.proto b/packages/proto/backupv1/server_to_agent.proto new file mode 100644 index 0000000..31c8629 --- /dev/null +++ b/packages/proto/backupv1/server_to_agent.proto @@ -0,0 +1,147 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Messages flowing from the server to the agent over the long-lived WSS +// connection. Each is wrapped in an Envelope (see envelope.proto). + +syntax = "proto3"; + +package backup.v1; + +import "backupv1/common.proto"; + +option go_package = "github.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1"; + +// ----------------------------------------------------------------------------- +// Register handshake response +// ----------------------------------------------------------------------------- + +// RegisterAck is the reply to Register. Carries an AgentConfig snapshot so the +// agent can begin work without an extra round-trip. +// See docs/07-api-contract.md §5 (RegisterAck). +message RegisterAck { + string session_id = 1; + AgentConfig config = 2; + uint32 heartbeat_interval_sec = 3; + string server_time_iso = 4; +} + +// ----------------------------------------------------------------------------- +// AgentConfig and sub-messages (also reused inside ConfigUpdate) +// ----------------------------------------------------------------------------- + +// AgentConfig is the full configuration snapshot for an agent. +// See docs/07-api-contract.md §5 (AgentConfig). +message AgentConfig { + uint64 version = 1; + repeated Target targets = 2; + repeated BackupJobSpec jobs = 3; + repeated HealthCheckSpec health_checks = 4; + MaintenanceWindow maintenance = 5; + AdvancedSettings advanced = 6; +} + +// Target is a single backup source (database) managed by this agent. +// See docs/07-api-contract.md §5 (Target). +message Target { + string id = 1; + DbType type = 2; + string display_name = 3; + ConnectionConfig connection = 4; +} + +// ConnectionConfig describes how the agent connects to a Target's database. +// See docs/07-api-contract.md §5 (ConnectionConfig). +message ConnectionConfig { + string host = 1; + uint32 port = 2; + string database = 3; + string username = 4; + string password_secret_ref = 5; // reference to encrypted secret in vault + string container_id = 6; // populated for DOCKER_EXEC strategy + ConnectionStrategy strategy = 7; +} + +// BackupJobSpec defines a scheduled backup job for one Target. +// See docs/07-api-contract.md §5 (BackupJobSpec). +message BackupJobSpec { + string id = 1; + string target_id = 2; + string cron = 3; // cron expression in UTC + uint32 retention_days = 4; + string compression = 5; // "zstd" | "gzip" | "none" + bool encryption_enabled = 6; + repeated string pre_hooks = 7; + repeated string post_hooks = 8; + RetryPolicy retry = 9; + uint32 timeout_sec = 10; +} + +// HealthCheckSpec defines a single health-check probe. +// See docs/07-api-contract.md §5 (HealthCheckSpec). +message HealthCheckSpec { + string id = 1; + string name = 2; + CheckType type = 3; + string target = 4; // URL or host:port + uint32 interval_sec = 5; + uint32 timeout_sec = 6; + uint32 expected_status = 7; // expected HTTP status (HTTP/HTTPS only) +} + +// ----------------------------------------------------------------------------- +// Config push +// ----------------------------------------------------------------------------- + +// ConfigUpdate replaces the agent's current AgentConfig. +// See docs/07-api-contract.md §5. +message ConfigUpdate { + AgentConfig config = 1; +} + +// ----------------------------------------------------------------------------- +// Job control +// ----------------------------------------------------------------------------- + +// RunBackup tells the agent to execute one backup run for the named job. +// Server pre-issues S3 upload credentials and a KMS-wrapped DEK. +// See docs/07-api-contract.md §5 (RunBackup). +message RunBackup { + string job_id = 1; + string run_id = 2; + bool manual_trigger = 3; + S3UploadCreds upload_creds = 4; + bytes encrypted_dek = 5; // DEK wrapped by KMS +} + +// CancelJob requests cancellation of an in-flight job. +// See docs/07-api-contract.md §5 (CancelJob). +message CancelJob { + string job_id = 1; +} + +// RunHealthCheck triggers an immediate, out-of-schedule health-check probe. +// See docs/07-api-contract.md §5 (RunHealthCheck). +message RunHealthCheck { + string check_id = 1; +} + +// ----------------------------------------------------------------------------- +// Lifecycle / liveness +// ----------------------------------------------------------------------------- + +// SelfUpdate asks the agent to upgrade its binary to target_version. +// cosign_signature lets the agent verify the binary before swapping it in. +// See docs/07-api-contract.md §5 (SelfUpdate). +message SelfUpdate { + string target_version = 1; + string binary_url = 2; + string sha256 = 3; + bytes cosign_signature = 4; + bool force = 5; +} + +// Ping is an application-level keepalive complementing WS-level pings. +// The agent replies with an Ack carrying the same correlation_id. +message Ping { + uint64 ts_ms = 1; +} diff --git a/packages/proto/buf.gen.yaml b/packages/proto/buf.gen.yaml new file mode 100644 index 0000000..45f9f80 --- /dev/null +++ b/packages/proto/buf.gen.yaml @@ -0,0 +1,18 @@ +# Backupy protobuf codegen — see docs/07-api-contract.md +# +# Targets: +# - Go bindings for apps/agent and apps/server. +# +# Run via `make gen` from the repo root (which `cd`s into this directory and +# invokes `buf generate`). Output is committed to gen/go/v1/. + +version: v2 + +managed: + enabled: true + +plugins: + - remote: buf.build/protocolbuffers/go + out: gen/go + opt: + - paths=source_relative diff --git a/packages/proto/buf.yaml b/packages/proto/buf.yaml new file mode 100644 index 0000000..6a9e5c9 --- /dev/null +++ b/packages/proto/buf.yaml @@ -0,0 +1,34 @@ +# Backupy protobuf module — see docs/07-api-contract.md +# +# This module owns the canonical wire-format definitions used by: +# - apps/agent (Go) +# - apps/server (Go) +# +# Generated code lives under gen/ and is committed (small, deterministic, +# trivially regenerated via `make proto`). + +version: v2 + +modules: + - path: . + +lint: + use: + - STANDARD + except: + # Enum values in common.proto (e.g. POSTGRESQL, TCP, HTTP) are intentionally + # short and unprefixed because they read naturally in generated Go code as + # backupv1.POSTGRESQL etc. STANDARD's ENUM_VALUE_PREFIX would require + # DB_TYPE_POSTGRESQL, which is needlessly verbose. + - ENUM_VALUE_PREFIX + - ENUM_ZERO_VALUE_SUFFIX + # We use proto messages over WSS, not gRPC services, so service-related + # rules do not apply. + - RPC_REQUEST_STANDARD_NAME + - RPC_RESPONSE_STANDARD_NAME + - RPC_REQUEST_RESPONSE_UNIQUE + - SERVICE_SUFFIX + +breaking: + use: + - FILE diff --git a/packages/proto/gen/.gitkeep b/packages/proto/gen/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/proto/gen/go/backupv1/agent_to_server.pb.go b/packages/proto/gen/go/backupv1/agent_to_server.pb.go new file mode 100644 index 0000000..ebcaaa3 --- /dev/null +++ b/packages/proto/gen/go/backupv1/agent_to_server.pb.go @@ -0,0 +1,997 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Messages flowing from the agent to the server over the long-lived WSS +// connection. Each is wrapped in an Envelope (see envelope.proto). + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: backupv1/agent_to_server.proto + +package backupv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Register is the first message the agent sends after authenticating the +// WSS connection. The server replies with RegisterAck. +// See docs/07-api-contract.md §4 (Register). +type Register struct { + state protoimpl.MessageState `protogen:"open.v1"` + AgentVersion string `protobuf:"bytes,1,opt,name=agent_version,json=agentVersion,proto3" json:"agent_version,omitempty"` + Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3" json:"hostname,omitempty"` + Os string `protobuf:"bytes,3,opt,name=os,proto3" json:"os,omitempty"` // "linux" + Arch string `protobuf:"bytes,4,opt,name=arch,proto3" json:"arch,omitempty"` // "amd64" + DockerVersion string `protobuf:"bytes,5,opt,name=docker_version,json=dockerVersion,proto3" json:"docker_version,omitempty"` + Capabilities []string `protobuf:"bytes,6,rep,name=capabilities,proto3" json:"capabilities,omitempty"` // ["pg_dump", "mysqldump", "docker_discovery"] + LastKnownConfigVersion uint64 `protobuf:"varint,7,opt,name=last_known_config_version,json=lastKnownConfigVersion,proto3" json:"last_known_config_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Register) Reset() { + *x = Register{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Register) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Register) ProtoMessage() {} + +func (x *Register) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Register.ProtoReflect.Descriptor instead. +func (*Register) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{0} +} + +func (x *Register) GetAgentVersion() string { + if x != nil { + return x.AgentVersion + } + return "" +} + +func (x *Register) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +func (x *Register) GetOs() string { + if x != nil { + return x.Os + } + return "" +} + +func (x *Register) GetArch() string { + if x != nil { + return x.Arch + } + return "" +} + +func (x *Register) GetDockerVersion() string { + if x != nil { + return x.DockerVersion + } + return "" +} + +func (x *Register) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *Register) GetLastKnownConfigVersion() uint64 { + if x != nil { + return x.LastKnownConfigVersion + } + return 0 +} + +// Heartbeat is sent every heartbeat_interval_sec (default 30s). +// See docs/07-api-contract.md §4 (Heartbeat). +type Heartbeat struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfigVersion uint64 `protobuf:"varint,1,opt,name=config_version,json=configVersion,proto3" json:"config_version,omitempty"` + Metrics *AgentMetrics `protobuf:"bytes,2,opt,name=metrics,proto3" json:"metrics,omitempty"` + ActiveJobIds []string `protobuf:"bytes,3,rep,name=active_job_ids,json=activeJobIds,proto3" json:"active_job_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Heartbeat) Reset() { + *x = Heartbeat{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Heartbeat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Heartbeat) ProtoMessage() {} + +func (x *Heartbeat) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. +func (*Heartbeat) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{1} +} + +func (x *Heartbeat) GetConfigVersion() uint64 { + if x != nil { + return x.ConfigVersion + } + return 0 +} + +func (x *Heartbeat) GetMetrics() *AgentMetrics { + if x != nil { + return x.Metrics + } + return nil +} + +func (x *Heartbeat) GetActiveJobIds() []string { + if x != nil { + return x.ActiveJobIds + } + return nil +} + +// DiscoveryReport is sent on startup and whenever docker events change the set +// of running containers. See docs/07-api-contract.md §4 (DiscoveryReport). +type DiscoveryReport struct { + state protoimpl.MessageState `protogen:"open.v1"` + Containers []*DiscoveredContainer `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DiscoveryReport) Reset() { + *x = DiscoveryReport{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DiscoveryReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DiscoveryReport) ProtoMessage() {} + +func (x *DiscoveryReport) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DiscoveryReport.ProtoReflect.Descriptor instead. +func (*DiscoveryReport) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{2} +} + +func (x *DiscoveryReport) GetContainers() []*DiscoveredContainer { + if x != nil { + return x.Containers + } + return nil +} + +type DiscoveredContainer struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContainerId string `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Image string `protobuf:"bytes,3,opt,name=image,proto3" json:"image,omitempty"` // "postgres:16" + DetectedDbType string `protobuf:"bytes,4,opt,name=detected_db_type,json=detectedDbType,proto3" json:"detected_db_type,omitempty"` // "postgresql" + Networks []string `protobuf:"bytes,5,rep,name=networks,proto3" json:"networks,omitempty"` + EnvHints map[string]string `protobuf:"bytes,6,rep,name=env_hints,json=envHints,proto3" json:"env_hints,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // env vars filtered to remove secrets + Ports []*PortBinding `protobuf:"bytes,7,rep,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DiscoveredContainer) Reset() { + *x = DiscoveredContainer{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DiscoveredContainer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DiscoveredContainer) ProtoMessage() {} + +func (x *DiscoveredContainer) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DiscoveredContainer.ProtoReflect.Descriptor instead. +func (*DiscoveredContainer) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{3} +} + +func (x *DiscoveredContainer) GetContainerId() string { + if x != nil { + return x.ContainerId + } + return "" +} + +func (x *DiscoveredContainer) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DiscoveredContainer) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + +func (x *DiscoveredContainer) GetDetectedDbType() string { + if x != nil { + return x.DetectedDbType + } + return "" +} + +func (x *DiscoveredContainer) GetNetworks() []string { + if x != nil { + return x.Networks + } + return nil +} + +func (x *DiscoveredContainer) GetEnvHints() map[string]string { + if x != nil { + return x.EnvHints + } + return nil +} + +func (x *DiscoveredContainer) GetPorts() []*PortBinding { + if x != nil { + return x.Ports + } + return nil +} + +// JobUpdate is a progress tick for a running backup job. +// See docs/07-api-contract.md §4 (JobUpdate). +// +// run_id was appended (field 6) after the initial Phase-1 wire so the +// scheduler can correlate a tick to a specific BackupRun row. Older +// agents that omit it are tolerated as "job-level" updates (no run row +// transition). +type JobUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Status JobStatus `protobuf:"varint,2,opt,name=status,proto3,enum=backup.v1.JobStatus" json:"status,omitempty"` + ProgressPercent uint32 `protobuf:"varint,3,opt,name=progress_percent,json=progressPercent,proto3" json:"progress_percent,omitempty"` + CurrentStep string `protobuf:"bytes,4,opt,name=current_step,json=currentStep,proto3" json:"current_step,omitempty"` // "dumping", "compressing", "uploading" + ErrorMessage string `protobuf:"bytes,5,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` // populated when status == FAILED + RunId string `protobuf:"bytes,6,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` // correlates to BackupRun.id; appended in v1.1 + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *JobUpdate) Reset() { + *x = JobUpdate{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *JobUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*JobUpdate) ProtoMessage() {} + +func (x *JobUpdate) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use JobUpdate.ProtoReflect.Descriptor instead. +func (*JobUpdate) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{4} +} + +func (x *JobUpdate) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *JobUpdate) GetStatus() JobStatus { + if x != nil { + return x.Status + } + return JobStatus_JOB_STATUS_UNSPECIFIED +} + +func (x *JobUpdate) GetProgressPercent() uint32 { + if x != nil { + return x.ProgressPercent + } + return 0 +} + +func (x *JobUpdate) GetCurrentStep() string { + if x != nil { + return x.CurrentStep + } + return "" +} + +func (x *JobUpdate) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +func (x *JobUpdate) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +// BackupCompleted is sent once after a successful S3 upload, carrying the +// metadata the server needs to persist a backup_runs row. +// See docs/07-api-contract.md §4 (BackupCompleted). +type BackupCompleted struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + S3Key string `protobuf:"bytes,3,opt,name=s3_key,json=s3Key,proto3" json:"s3_key,omitempty"` + SizeBytes uint64 `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Sha256 string `protobuf:"bytes,5,opt,name=sha256,proto3" json:"sha256,omitempty"` + DurationMs uint64 `protobuf:"varint,6,opt,name=duration_ms,json=durationMs,proto3" json:"duration_ms,omitempty"` + DekKmsId string `protobuf:"bytes,7,opt,name=dek_kms_id,json=dekKmsId,proto3" json:"dek_kms_id,omitempty"` + EncryptedDek []byte `protobuf:"bytes,8,opt,name=encrypted_dek,json=encryptedDek,proto3" json:"encrypted_dek,omitempty"` + Compression string `protobuf:"bytes,9,opt,name=compression,proto3" json:"compression,omitempty"` // "zstd" + DbEngineVersion string `protobuf:"bytes,10,opt,name=db_engine_version,json=dbEngineVersion,proto3" json:"db_engine_version,omitempty"` // "PostgreSQL 16.2" + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupCompleted) Reset() { + *x = BackupCompleted{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupCompleted) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupCompleted) ProtoMessage() {} + +func (x *BackupCompleted) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupCompleted.ProtoReflect.Descriptor instead. +func (*BackupCompleted) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{5} +} + +func (x *BackupCompleted) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *BackupCompleted) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *BackupCompleted) GetS3Key() string { + if x != nil { + return x.S3Key + } + return "" +} + +func (x *BackupCompleted) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *BackupCompleted) GetSha256() string { + if x != nil { + return x.Sha256 + } + return "" +} + +func (x *BackupCompleted) GetDurationMs() uint64 { + if x != nil { + return x.DurationMs + } + return 0 +} + +func (x *BackupCompleted) GetDekKmsId() string { + if x != nil { + return x.DekKmsId + } + return "" +} + +func (x *BackupCompleted) GetEncryptedDek() []byte { + if x != nil { + return x.EncryptedDek + } + return nil +} + +func (x *BackupCompleted) GetCompression() string { + if x != nil { + return x.Compression + } + return "" +} + +func (x *BackupCompleted) GetDbEngineVersion() string { + if x != nil { + return x.DbEngineVersion + } + return "" +} + +// HealthCheckResult is the outcome of a single probe run. +// See docs/07-api-contract.md §4 (HealthCheckResult). +type HealthCheckResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + CheckId string `protobuf:"bytes,1,opt,name=check_id,json=checkId,proto3" json:"check_id,omitempty"` + TsMs uint64 `protobuf:"varint,2,opt,name=ts_ms,json=tsMs,proto3" json:"ts_ms,omitempty"` + Ok bool `protobuf:"varint,3,opt,name=ok,proto3" json:"ok,omitempty"` + LatencyMs uint32 `protobuf:"varint,4,opt,name=latency_ms,json=latencyMs,proto3" json:"latency_ms,omitempty"` + Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"` + StatusCode uint32 `protobuf:"varint,6,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` // populated for HTTP/HTTPS probes + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthCheckResult) Reset() { + *x = HealthCheckResult{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthCheckResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthCheckResult) ProtoMessage() {} + +func (x *HealthCheckResult) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthCheckResult.ProtoReflect.Descriptor instead. +func (*HealthCheckResult) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{6} +} + +func (x *HealthCheckResult) GetCheckId() string { + if x != nil { + return x.CheckId + } + return "" +} + +func (x *HealthCheckResult) GetTsMs() uint64 { + if x != nil { + return x.TsMs + } + return 0 +} + +func (x *HealthCheckResult) GetOk() bool { + if x != nil { + return x.Ok + } + return false +} + +func (x *HealthCheckResult) GetLatencyMs() uint32 { + if x != nil { + return x.LatencyMs + } + return 0 +} + +func (x *HealthCheckResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *HealthCheckResult) GetStatusCode() uint32 { + if x != nil { + return x.StatusCode + } + return 0 +} + +// LogEvent is a structured log line streamed from the agent. +// See docs/07-api-contract.md §4 (LogEvent). +type LogEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + TsMs uint64 `protobuf:"varint,1,opt,name=ts_ms,json=tsMs,proto3" json:"ts_ms,omitempty"` + Level LogLevel `protobuf:"varint,2,opt,name=level,proto3,enum=backup.v1.LogLevel" json:"level,omitempty"` + JobId string `protobuf:"bytes,3,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` // optional + Message string `protobuf:"bytes,4,opt,name=message,proto3" json:"message,omitempty"` + Fields map[string]string `protobuf:"bytes,5,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LogEvent) Reset() { + *x = LogEvent{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogEvent) ProtoMessage() {} + +func (x *LogEvent) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogEvent.ProtoReflect.Descriptor instead. +func (*LogEvent) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{7} +} + +func (x *LogEvent) GetTsMs() uint64 { + if x != nil { + return x.TsMs + } + return 0 +} + +func (x *LogEvent) GetLevel() LogLevel { + if x != nil { + return x.Level + } + return LogLevel_TRACE +} + +func (x *LogEvent) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *LogEvent) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *LogEvent) GetFields() map[string]string { + if x != nil { + return x.Fields + } + return nil +} + +// RestoreUpdate is reserved for future agent-driven restore flows. In MVP the +// agent never receives a RunRestore command — restore happens through the REST +// download endpoint and the local backupy-decrypt CLI. Kept in the contract so +// the field number is stable for Phase 3. +// See docs/07-api-contract.md §4 (RestoreUpdate). +type RestoreUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + RestoreId string `protobuf:"bytes,1,opt,name=restore_id,json=restoreId,proto3" json:"restore_id,omitempty"` + Status JobStatus `protobuf:"varint,2,opt,name=status,proto3,enum=backup.v1.JobStatus" json:"status,omitempty"` + ProgressPercent uint32 `protobuf:"varint,3,opt,name=progress_percent,json=progressPercent,proto3" json:"progress_percent,omitempty"` + CurrentStep string `protobuf:"bytes,4,opt,name=current_step,json=currentStep,proto3" json:"current_step,omitempty"` + ErrorMessage string `protobuf:"bytes,5,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RestoreUpdate) Reset() { + *x = RestoreUpdate{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RestoreUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RestoreUpdate) ProtoMessage() {} + +func (x *RestoreUpdate) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RestoreUpdate.ProtoReflect.Descriptor instead. +func (*RestoreUpdate) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{8} +} + +func (x *RestoreUpdate) GetRestoreId() string { + if x != nil { + return x.RestoreId + } + return "" +} + +func (x *RestoreUpdate) GetStatus() JobStatus { + if x != nil { + return x.Status + } + return JobStatus_JOB_STATUS_UNSPECIFIED +} + +func (x *RestoreUpdate) GetProgressPercent() uint32 { + if x != nil { + return x.ProgressPercent + } + return 0 +} + +func (x *RestoreUpdate) GetCurrentStep() string { + if x != nil { + return x.CurrentStep + } + return "" +} + +func (x *RestoreUpdate) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +// Ack responds to a server message identified by correlation_id. +// See docs/07-api-contract.md §4 (Ack). +type Ack struct { + state protoimpl.MessageState `protogen:"open.v1"` + CorrelationId string `protobuf:"bytes,1,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` + Accepted bool `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"` + Reason string `protobuf:"bytes,3,opt,name=reason,proto3" json:"reason,omitempty"` // populated when accepted == false + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ack) Reset() { + *x = Ack{} + mi := &file_backupv1_agent_to_server_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ack) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ack) ProtoMessage() {} + +func (x *Ack) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_agent_to_server_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ack.ProtoReflect.Descriptor instead. +func (*Ack) Descriptor() ([]byte, []int) { + return file_backupv1_agent_to_server_proto_rawDescGZIP(), []int{9} +} + +func (x *Ack) GetCorrelationId() string { + if x != nil { + return x.CorrelationId + } + return "" +} + +func (x *Ack) GetAccepted() bool { + if x != nil { + return x.Accepted + } + return false +} + +func (x *Ack) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +var File_backupv1_agent_to_server_proto protoreflect.FileDescriptor + +const file_backupv1_agent_to_server_proto_rawDesc = "" + + "\n" + + "\x1ebackupv1/agent_to_server.proto\x12\tbackup.v1\x1a\x15backupv1/common.proto\"\xf5\x01\n" + + "\bRegister\x12#\n" + + "\ragent_version\x18\x01 \x01(\tR\fagentVersion\x12\x1a\n" + + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x0e\n" + + "\x02os\x18\x03 \x01(\tR\x02os\x12\x12\n" + + "\x04arch\x18\x04 \x01(\tR\x04arch\x12%\n" + + "\x0edocker_version\x18\x05 \x01(\tR\rdockerVersion\x12\"\n" + + "\fcapabilities\x18\x06 \x03(\tR\fcapabilities\x129\n" + + "\x19last_known_config_version\x18\a \x01(\x04R\x16lastKnownConfigVersion\"\x8b\x01\n" + + "\tHeartbeat\x12%\n" + + "\x0econfig_version\x18\x01 \x01(\x04R\rconfigVersion\x121\n" + + "\ametrics\x18\x02 \x01(\v2\x17.backup.v1.AgentMetricsR\ametrics\x12$\n" + + "\x0eactive_job_ids\x18\x03 \x03(\tR\factiveJobIds\"Q\n" + + "\x0fDiscoveryReport\x12>\n" + + "\n" + + "containers\x18\x01 \x03(\v2\x1e.backup.v1.DiscoveredContainerR\n" + + "containers\"\xde\x02\n" + + "\x13DiscoveredContainer\x12!\n" + + "\fcontainer_id\x18\x01 \x01(\tR\vcontainerId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x14\n" + + "\x05image\x18\x03 \x01(\tR\x05image\x12(\n" + + "\x10detected_db_type\x18\x04 \x01(\tR\x0edetectedDbType\x12\x1a\n" + + "\bnetworks\x18\x05 \x03(\tR\bnetworks\x12I\n" + + "\tenv_hints\x18\x06 \x03(\v2,.backup.v1.DiscoveredContainer.EnvHintsEntryR\benvHints\x12,\n" + + "\x05ports\x18\a \x03(\v2\x16.backup.v1.PortBindingR\x05ports\x1a;\n" + + "\rEnvHintsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xda\x01\n" + + "\tJobUpdate\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\x12,\n" + + "\x06status\x18\x02 \x01(\x0e2\x14.backup.v1.JobStatusR\x06status\x12)\n" + + "\x10progress_percent\x18\x03 \x01(\rR\x0fprogressPercent\x12!\n" + + "\fcurrent_step\x18\x04 \x01(\tR\vcurrentStep\x12#\n" + + "\rerror_message\x18\x05 \x01(\tR\ferrorMessage\x12\x15\n" + + "\x06run_id\x18\x06 \x01(\tR\x05runId\"\xbf\x02\n" + + "\x0fBackupCompleted\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x15\n" + + "\x06run_id\x18\x02 \x01(\tR\x05runId\x12\x15\n" + + "\x06s3_key\x18\x03 \x01(\tR\x05s3Key\x12\x1d\n" + + "\n" + + "size_bytes\x18\x04 \x01(\x04R\tsizeBytes\x12\x16\n" + + "\x06sha256\x18\x05 \x01(\tR\x06sha256\x12\x1f\n" + + "\vduration_ms\x18\x06 \x01(\x04R\n" + + "durationMs\x12\x1c\n" + + "\n" + + "dek_kms_id\x18\a \x01(\tR\bdekKmsId\x12#\n" + + "\rencrypted_dek\x18\b \x01(\fR\fencryptedDek\x12 \n" + + "\vcompression\x18\t \x01(\tR\vcompression\x12*\n" + + "\x11db_engine_version\x18\n" + + " \x01(\tR\x0fdbEngineVersion\"\xa9\x01\n" + + "\x11HealthCheckResult\x12\x19\n" + + "\bcheck_id\x18\x01 \x01(\tR\acheckId\x12\x13\n" + + "\x05ts_ms\x18\x02 \x01(\x04R\x04tsMs\x12\x0e\n" + + "\x02ok\x18\x03 \x01(\bR\x02ok\x12\x1d\n" + + "\n" + + "latency_ms\x18\x04 \x01(\rR\tlatencyMs\x12\x14\n" + + "\x05error\x18\x05 \x01(\tR\x05error\x12\x1f\n" + + "\vstatus_code\x18\x06 \x01(\rR\n" + + "statusCode\"\xef\x01\n" + + "\bLogEvent\x12\x13\n" + + "\x05ts_ms\x18\x01 \x01(\x04R\x04tsMs\x12)\n" + + "\x05level\x18\x02 \x01(\x0e2\x13.backup.v1.LogLevelR\x05level\x12\x15\n" + + "\x06job_id\x18\x03 \x01(\tR\x05jobId\x12\x18\n" + + "\amessage\x18\x04 \x01(\tR\amessage\x127\n" + + "\x06fields\x18\x05 \x03(\v2\x1f.backup.v1.LogEvent.FieldsEntryR\x06fields\x1a9\n" + + "\vFieldsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xcf\x01\n" + + "\rRestoreUpdate\x12\x1d\n" + + "\n" + + "restore_id\x18\x01 \x01(\tR\trestoreId\x12,\n" + + "\x06status\x18\x02 \x01(\x0e2\x14.backup.v1.JobStatusR\x06status\x12)\n" + + "\x10progress_percent\x18\x03 \x01(\rR\x0fprogressPercent\x12!\n" + + "\fcurrent_step\x18\x04 \x01(\tR\vcurrentStep\x12#\n" + + "\rerror_message\x18\x05 \x01(\tR\ferrorMessage\"`\n" + + "\x03Ack\x12%\n" + + "\x0ecorrelation_id\x18\x01 \x01(\tR\rcorrelationId\x12\x1a\n" + + "\baccepted\x18\x02 \x01(\bR\baccepted\x12\x16\n" + + "\x06reason\x18\x03 \x01(\tR\x06reasonB\xac\x01\n" + + "\rcom.backup.v1B\x12AgentToServerProtoP\x01ZBgithub.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1\xa2\x02\x03BXX\xaa\x02\tBackup.V1\xca\x02\tBackup\\V1\xe2\x02\x15Backup\\V1\\GPBMetadata\xea\x02\n" + + "Backup::V1b\x06proto3" + +var ( + file_backupv1_agent_to_server_proto_rawDescOnce sync.Once + file_backupv1_agent_to_server_proto_rawDescData []byte +) + +func file_backupv1_agent_to_server_proto_rawDescGZIP() []byte { + file_backupv1_agent_to_server_proto_rawDescOnce.Do(func() { + file_backupv1_agent_to_server_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_backupv1_agent_to_server_proto_rawDesc), len(file_backupv1_agent_to_server_proto_rawDesc))) + }) + return file_backupv1_agent_to_server_proto_rawDescData +} + +var file_backupv1_agent_to_server_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_backupv1_agent_to_server_proto_goTypes = []any{ + (*Register)(nil), // 0: backup.v1.Register + (*Heartbeat)(nil), // 1: backup.v1.Heartbeat + (*DiscoveryReport)(nil), // 2: backup.v1.DiscoveryReport + (*DiscoveredContainer)(nil), // 3: backup.v1.DiscoveredContainer + (*JobUpdate)(nil), // 4: backup.v1.JobUpdate + (*BackupCompleted)(nil), // 5: backup.v1.BackupCompleted + (*HealthCheckResult)(nil), // 6: backup.v1.HealthCheckResult + (*LogEvent)(nil), // 7: backup.v1.LogEvent + (*RestoreUpdate)(nil), // 8: backup.v1.RestoreUpdate + (*Ack)(nil), // 9: backup.v1.Ack + nil, // 10: backup.v1.DiscoveredContainer.EnvHintsEntry + nil, // 11: backup.v1.LogEvent.FieldsEntry + (*AgentMetrics)(nil), // 12: backup.v1.AgentMetrics + (*PortBinding)(nil), // 13: backup.v1.PortBinding + (JobStatus)(0), // 14: backup.v1.JobStatus + (LogLevel)(0), // 15: backup.v1.LogLevel +} +var file_backupv1_agent_to_server_proto_depIdxs = []int32{ + 12, // 0: backup.v1.Heartbeat.metrics:type_name -> backup.v1.AgentMetrics + 3, // 1: backup.v1.DiscoveryReport.containers:type_name -> backup.v1.DiscoveredContainer + 10, // 2: backup.v1.DiscoveredContainer.env_hints:type_name -> backup.v1.DiscoveredContainer.EnvHintsEntry + 13, // 3: backup.v1.DiscoveredContainer.ports:type_name -> backup.v1.PortBinding + 14, // 4: backup.v1.JobUpdate.status:type_name -> backup.v1.JobStatus + 15, // 5: backup.v1.LogEvent.level:type_name -> backup.v1.LogLevel + 11, // 6: backup.v1.LogEvent.fields:type_name -> backup.v1.LogEvent.FieldsEntry + 14, // 7: backup.v1.RestoreUpdate.status:type_name -> backup.v1.JobStatus + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_backupv1_agent_to_server_proto_init() } +func file_backupv1_agent_to_server_proto_init() { + if File_backupv1_agent_to_server_proto != nil { + return + } + file_backupv1_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_backupv1_agent_to_server_proto_rawDesc), len(file_backupv1_agent_to_server_proto_rawDesc)), + NumEnums: 0, + NumMessages: 12, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_backupv1_agent_to_server_proto_goTypes, + DependencyIndexes: file_backupv1_agent_to_server_proto_depIdxs, + MessageInfos: file_backupv1_agent_to_server_proto_msgTypes, + }.Build() + File_backupv1_agent_to_server_proto = out.File + file_backupv1_agent_to_server_proto_goTypes = nil + file_backupv1_agent_to_server_proto_depIdxs = nil +} diff --git a/packages/proto/gen/go/backupv1/common.pb.go b/packages/proto/gen/go/backupv1/common.pb.go new file mode 100644 index 0000000..39035ee --- /dev/null +++ b/packages/proto/gen/go/backupv1/common.pb.go @@ -0,0 +1,886 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Shared enums and messages used across agent_to_server and server_to_agent +// payloads. Keep this file dependency-free — only primitives and other +// definitions from this same file may appear here. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: backupv1/common.proto + +package backupv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// JobStatus is the lifecycle state for a backup or restore job. +// See docs/07-api-contract.md §4 (JobUpdate, RestoreUpdate). +type JobStatus int32 + +const ( + JobStatus_JOB_STATUS_UNSPECIFIED JobStatus = 0 + JobStatus_QUEUED JobStatus = 1 + JobStatus_RUNNING JobStatus = 2 + JobStatus_SUCCESS JobStatus = 3 + JobStatus_FAILED JobStatus = 4 + JobStatus_CANCELLED JobStatus = 5 +) + +// Enum value maps for JobStatus. +var ( + JobStatus_name = map[int32]string{ + 0: "JOB_STATUS_UNSPECIFIED", + 1: "QUEUED", + 2: "RUNNING", + 3: "SUCCESS", + 4: "FAILED", + 5: "CANCELLED", + } + JobStatus_value = map[string]int32{ + "JOB_STATUS_UNSPECIFIED": 0, + "QUEUED": 1, + "RUNNING": 2, + "SUCCESS": 3, + "FAILED": 4, + "CANCELLED": 5, + } +) + +func (x JobStatus) Enum() *JobStatus { + p := new(JobStatus) + *p = x + return p +} + +func (x JobStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (JobStatus) Descriptor() protoreflect.EnumDescriptor { + return file_backupv1_common_proto_enumTypes[0].Descriptor() +} + +func (JobStatus) Type() protoreflect.EnumType { + return &file_backupv1_common_proto_enumTypes[0] +} + +func (x JobStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use JobStatus.Descriptor instead. +func (JobStatus) EnumDescriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{0} +} + +// DbType enumerates supported database engines for targets. +// See docs/07-api-contract.md §5 (Target). +type DbType int32 + +const ( + DbType_DB_UNSPECIFIED DbType = 0 + DbType_POSTGRESQL DbType = 1 + DbType_MYSQL DbType = 2 + DbType_MARIADB DbType = 3 + DbType_MONGODB DbType = 4 + DbType_REDIS DbType = 5 + DbType_SQLITE DbType = 6 +) + +// Enum value maps for DbType. +var ( + DbType_name = map[int32]string{ + 0: "DB_UNSPECIFIED", + 1: "POSTGRESQL", + 2: "MYSQL", + 3: "MARIADB", + 4: "MONGODB", + 5: "REDIS", + 6: "SQLITE", + } + DbType_value = map[string]int32{ + "DB_UNSPECIFIED": 0, + "POSTGRESQL": 1, + "MYSQL": 2, + "MARIADB": 3, + "MONGODB": 4, + "REDIS": 5, + "SQLITE": 6, + } +) + +func (x DbType) Enum() *DbType { + p := new(DbType) + *p = x + return p +} + +func (x DbType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DbType) Descriptor() protoreflect.EnumDescriptor { + return file_backupv1_common_proto_enumTypes[1].Descriptor() +} + +func (DbType) Type() protoreflect.EnumType { + return &file_backupv1_common_proto_enumTypes[1] +} + +func (x DbType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DbType.Descriptor instead. +func (DbType) EnumDescriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{1} +} + +// ConnectionStrategy is how the agent reaches the database. +// See docs/07-api-contract.md §5 (ConnectionConfig). +type ConnectionStrategy int32 + +const ( + ConnectionStrategy_TCP ConnectionStrategy = 0 + ConnectionStrategy_DOCKER_EXEC ConnectionStrategy = 1 + ConnectionStrategy_UNIX_SOCKET ConnectionStrategy = 2 +) + +// Enum value maps for ConnectionStrategy. +var ( + ConnectionStrategy_name = map[int32]string{ + 0: "TCP", + 1: "DOCKER_EXEC", + 2: "UNIX_SOCKET", + } + ConnectionStrategy_value = map[string]int32{ + "TCP": 0, + "DOCKER_EXEC": 1, + "UNIX_SOCKET": 2, + } +) + +func (x ConnectionStrategy) Enum() *ConnectionStrategy { + p := new(ConnectionStrategy) + *p = x + return p +} + +func (x ConnectionStrategy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ConnectionStrategy) Descriptor() protoreflect.EnumDescriptor { + return file_backupv1_common_proto_enumTypes[2].Descriptor() +} + +func (ConnectionStrategy) Type() protoreflect.EnumType { + return &file_backupv1_common_proto_enumTypes[2] +} + +func (x ConnectionStrategy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ConnectionStrategy.Descriptor instead. +func (ConnectionStrategy) EnumDescriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{2} +} + +// CheckType enumerates supported health-check probe types. +// See docs/07-api-contract.md §5 (HealthCheckSpec). +type CheckType int32 + +const ( + CheckType_HTTP CheckType = 0 + CheckType_HTTPS CheckType = 1 + CheckType_TCP_CHECK CheckType = 2 +) + +// Enum value maps for CheckType. +var ( + CheckType_name = map[int32]string{ + 0: "HTTP", + 1: "HTTPS", + 2: "TCP_CHECK", + } + CheckType_value = map[string]int32{ + "HTTP": 0, + "HTTPS": 1, + "TCP_CHECK": 2, + } +) + +func (x CheckType) Enum() *CheckType { + p := new(CheckType) + *p = x + return p +} + +func (x CheckType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CheckType) Descriptor() protoreflect.EnumDescriptor { + return file_backupv1_common_proto_enumTypes[3].Descriptor() +} + +func (CheckType) Type() protoreflect.EnumType { + return &file_backupv1_common_proto_enumTypes[3] +} + +func (x CheckType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CheckType.Descriptor instead. +func (CheckType) EnumDescriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{3} +} + +// LogLevel is the severity for LogEvent records streamed by the agent. +// See docs/07-api-contract.md §4 (LogEvent). +type LogLevel int32 + +const ( + LogLevel_TRACE LogLevel = 0 + LogLevel_DEBUG LogLevel = 1 + LogLevel_INFO LogLevel = 2 + LogLevel_WARN LogLevel = 3 + LogLevel_ERROR LogLevel = 4 +) + +// Enum value maps for LogLevel. +var ( + LogLevel_name = map[int32]string{ + 0: "TRACE", + 1: "DEBUG", + 2: "INFO", + 3: "WARN", + 4: "ERROR", + } + LogLevel_value = map[string]int32{ + "TRACE": 0, + "DEBUG": 1, + "INFO": 2, + "WARN": 3, + "ERROR": 4, + } +) + +func (x LogLevel) Enum() *LogLevel { + p := new(LogLevel) + *p = x + return p +} + +func (x LogLevel) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LogLevel) Descriptor() protoreflect.EnumDescriptor { + return file_backupv1_common_proto_enumTypes[4].Descriptor() +} + +func (LogLevel) Type() protoreflect.EnumType { + return &file_backupv1_common_proto_enumTypes[4] +} + +func (x LogLevel) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use LogLevel.Descriptor instead. +func (LogLevel) EnumDescriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{4} +} + +// AgentMetrics is the snapshot of host resource usage attached to Heartbeat. +// See docs/07-api-contract.md §4 (Heartbeat). +type AgentMetrics struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuPercent float32 `protobuf:"fixed32,1,opt,name=cpu_percent,json=cpuPercent,proto3" json:"cpu_percent,omitempty"` + MemUsedBytes uint64 `protobuf:"varint,2,opt,name=mem_used_bytes,json=memUsedBytes,proto3" json:"mem_used_bytes,omitempty"` + MemTotalBytes uint64 `protobuf:"varint,3,opt,name=mem_total_bytes,json=memTotalBytes,proto3" json:"mem_total_bytes,omitempty"` + DiskUsedBytes uint64 `protobuf:"varint,4,opt,name=disk_used_bytes,json=diskUsedBytes,proto3" json:"disk_used_bytes,omitempty"` + DiskTotalBytes uint64 `protobuf:"varint,5,opt,name=disk_total_bytes,json=diskTotalBytes,proto3" json:"disk_total_bytes,omitempty"` + QueueDepth uint32 `protobuf:"varint,6,opt,name=queue_depth,json=queueDepth,proto3" json:"queue_depth,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AgentMetrics) Reset() { + *x = AgentMetrics{} + mi := &file_backupv1_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AgentMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AgentMetrics) ProtoMessage() {} + +func (x *AgentMetrics) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AgentMetrics.ProtoReflect.Descriptor instead. +func (*AgentMetrics) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{0} +} + +func (x *AgentMetrics) GetCpuPercent() float32 { + if x != nil { + return x.CpuPercent + } + return 0 +} + +func (x *AgentMetrics) GetMemUsedBytes() uint64 { + if x != nil { + return x.MemUsedBytes + } + return 0 +} + +func (x *AgentMetrics) GetMemTotalBytes() uint64 { + if x != nil { + return x.MemTotalBytes + } + return 0 +} + +func (x *AgentMetrics) GetDiskUsedBytes() uint64 { + if x != nil { + return x.DiskUsedBytes + } + return 0 +} + +func (x *AgentMetrics) GetDiskTotalBytes() uint64 { + if x != nil { + return x.DiskTotalBytes + } + return 0 +} + +func (x *AgentMetrics) GetQueueDepth() uint32 { + if x != nil { + return x.QueueDepth + } + return 0 +} + +// PortBinding describes a single container port published on the host. +// See docs/07-api-contract.md §4 (DiscoveryReport). +type PortBinding struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContainerPort uint32 `protobuf:"varint,1,opt,name=container_port,json=containerPort,proto3" json:"container_port,omitempty"` + HostPort uint32 `protobuf:"varint,2,opt,name=host_port,json=hostPort,proto3" json:"host_port,omitempty"` + Protocol string `protobuf:"bytes,3,opt,name=protocol,proto3" json:"protocol,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PortBinding) Reset() { + *x = PortBinding{} + mi := &file_backupv1_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PortBinding) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PortBinding) ProtoMessage() {} + +func (x *PortBinding) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PortBinding.ProtoReflect.Descriptor instead. +func (*PortBinding) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{1} +} + +func (x *PortBinding) GetContainerPort() uint32 { + if x != nil { + return x.ContainerPort + } + return 0 +} + +func (x *PortBinding) GetHostPort() uint32 { + if x != nil { + return x.HostPort + } + return 0 +} + +func (x *PortBinding) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + +// S3UploadCreds is a presigned PUT URL the agent uses to upload the encrypted +// backup blob. See docs/07-api-contract.md §5 (RunBackup). +type S3UploadCreds struct { + state protoimpl.MessageState `protogen:"open.v1"` + PresignedPutUrl string `protobuf:"bytes,1,opt,name=presigned_put_url,json=presignedPutUrl,proto3" json:"presigned_put_url,omitempty"` + ExpiresAtMs uint64 `protobuf:"varint,2,opt,name=expires_at_ms,json=expiresAtMs,proto3" json:"expires_at_ms,omitempty"` + FinalS3Key string `protobuf:"bytes,3,opt,name=final_s3_key,json=finalS3Key,proto3" json:"final_s3_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *S3UploadCreds) Reset() { + *x = S3UploadCreds{} + mi := &file_backupv1_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *S3UploadCreds) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3UploadCreds) ProtoMessage() {} + +func (x *S3UploadCreds) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3UploadCreds.ProtoReflect.Descriptor instead. +func (*S3UploadCreds) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{2} +} + +func (x *S3UploadCreds) GetPresignedPutUrl() string { + if x != nil { + return x.PresignedPutUrl + } + return "" +} + +func (x *S3UploadCreds) GetExpiresAtMs() uint64 { + if x != nil { + return x.ExpiresAtMs + } + return 0 +} + +func (x *S3UploadCreds) GetFinalS3Key() string { + if x != nil { + return x.FinalS3Key + } + return "" +} + +// S3DownloadCreds is a presigned GET URL for fetching a backup blob (reserved +// for future restore-via-agent flows). Not used in MVP — Phase 3. +type S3DownloadCreds struct { + state protoimpl.MessageState `protogen:"open.v1"` + PresignedGetUrl string `protobuf:"bytes,1,opt,name=presigned_get_url,json=presignedGetUrl,proto3" json:"presigned_get_url,omitempty"` + ExpiresAtMs uint64 `protobuf:"varint,2,opt,name=expires_at_ms,json=expiresAtMs,proto3" json:"expires_at_ms,omitempty"` + S3Key string `protobuf:"bytes,3,opt,name=s3_key,json=s3Key,proto3" json:"s3_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *S3DownloadCreds) Reset() { + *x = S3DownloadCreds{} + mi := &file_backupv1_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *S3DownloadCreds) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*S3DownloadCreds) ProtoMessage() {} + +func (x *S3DownloadCreds) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use S3DownloadCreds.ProtoReflect.Descriptor instead. +func (*S3DownloadCreds) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{3} +} + +func (x *S3DownloadCreds) GetPresignedGetUrl() string { + if x != nil { + return x.PresignedGetUrl + } + return "" +} + +func (x *S3DownloadCreds) GetExpiresAtMs() uint64 { + if x != nil { + return x.ExpiresAtMs + } + return 0 +} + +func (x *S3DownloadCreds) GetS3Key() string { + if x != nil { + return x.S3Key + } + return "" +} + +// RetryPolicy controls how the agent retries a failed backup job. +// See docs/07-api-contract.md §5 (BackupJobSpec). +type RetryPolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"` + BackoffSeconds []uint32 `protobuf:"varint,2,rep,packed,name=backoff_seconds,json=backoffSeconds,proto3" json:"backoff_seconds,omitempty"` // e.g. [60, 300, 1800] + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RetryPolicy) Reset() { + *x = RetryPolicy{} + mi := &file_backupv1_common_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RetryPolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RetryPolicy) ProtoMessage() {} + +func (x *RetryPolicy) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RetryPolicy.ProtoReflect.Descriptor instead. +func (*RetryPolicy) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{4} +} + +func (x *RetryPolicy) GetMaxAttempts() uint32 { + if x != nil { + return x.MaxAttempts + } + return 0 +} + +func (x *RetryPolicy) GetBackoffSeconds() []uint32 { + if x != nil { + return x.BackoffSeconds + } + return nil +} + +// MaintenanceWindow defines a UTC time-of-day window where backups are paused. +// See docs/07-api-contract.md §5 (AgentConfig). +type MaintenanceWindow struct { + state protoimpl.MessageState `protogen:"open.v1"` + StartUtc string `protobuf:"bytes,1,opt,name=start_utc,json=startUtc,proto3" json:"start_utc,omitempty"` // "02:00" + EndUtc string `protobuf:"bytes,2,opt,name=end_utc,json=endUtc,proto3" json:"end_utc,omitempty"` // "05:00" + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MaintenanceWindow) Reset() { + *x = MaintenanceWindow{} + mi := &file_backupv1_common_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MaintenanceWindow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MaintenanceWindow) ProtoMessage() {} + +func (x *MaintenanceWindow) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MaintenanceWindow.ProtoReflect.Descriptor instead. +func (*MaintenanceWindow) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{5} +} + +func (x *MaintenanceWindow) GetStartUtc() string { + if x != nil { + return x.StartUtc + } + return "" +} + +func (x *MaintenanceWindow) GetEndUtc() string { + if x != nil { + return x.EndUtc + } + return "" +} + +// AdvancedSettings carries miscellaneous tunables pushed in AgentConfig. +// See docs/07-api-contract.md §5 (AgentConfig). +type AdvancedSettings struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxParallelJobs uint32 `protobuf:"varint,1,opt,name=max_parallel_jobs,json=maxParallelJobs,proto3" json:"max_parallel_jobs,omitempty"` + LogLevel string `protobuf:"bytes,2,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"` + AutoUpdateEnabled bool `protobuf:"varint,3,opt,name=auto_update_enabled,json=autoUpdateEnabled,proto3" json:"auto_update_enabled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AdvancedSettings) Reset() { + *x = AdvancedSettings{} + mi := &file_backupv1_common_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AdvancedSettings) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AdvancedSettings) ProtoMessage() {} + +func (x *AdvancedSettings) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_common_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AdvancedSettings.ProtoReflect.Descriptor instead. +func (*AdvancedSettings) Descriptor() ([]byte, []int) { + return file_backupv1_common_proto_rawDescGZIP(), []int{6} +} + +func (x *AdvancedSettings) GetMaxParallelJobs() uint32 { + if x != nil { + return x.MaxParallelJobs + } + return 0 +} + +func (x *AdvancedSettings) GetLogLevel() string { + if x != nil { + return x.LogLevel + } + return "" +} + +func (x *AdvancedSettings) GetAutoUpdateEnabled() bool { + if x != nil { + return x.AutoUpdateEnabled + } + return false +} + +var File_backupv1_common_proto protoreflect.FileDescriptor + +const file_backupv1_common_proto_rawDesc = "" + + "\n" + + "\x15backupv1/common.proto\x12\tbackup.v1\"\xf0\x01\n" + + "\fAgentMetrics\x12\x1f\n" + + "\vcpu_percent\x18\x01 \x01(\x02R\n" + + "cpuPercent\x12$\n" + + "\x0emem_used_bytes\x18\x02 \x01(\x04R\fmemUsedBytes\x12&\n" + + "\x0fmem_total_bytes\x18\x03 \x01(\x04R\rmemTotalBytes\x12&\n" + + "\x0fdisk_used_bytes\x18\x04 \x01(\x04R\rdiskUsedBytes\x12(\n" + + "\x10disk_total_bytes\x18\x05 \x01(\x04R\x0ediskTotalBytes\x12\x1f\n" + + "\vqueue_depth\x18\x06 \x01(\rR\n" + + "queueDepth\"m\n" + + "\vPortBinding\x12%\n" + + "\x0econtainer_port\x18\x01 \x01(\rR\rcontainerPort\x12\x1b\n" + + "\thost_port\x18\x02 \x01(\rR\bhostPort\x12\x1a\n" + + "\bprotocol\x18\x03 \x01(\tR\bprotocol\"\x81\x01\n" + + "\rS3UploadCreds\x12*\n" + + "\x11presigned_put_url\x18\x01 \x01(\tR\x0fpresignedPutUrl\x12\"\n" + + "\rexpires_at_ms\x18\x02 \x01(\x04R\vexpiresAtMs\x12 \n" + + "\ffinal_s3_key\x18\x03 \x01(\tR\n" + + "finalS3Key\"x\n" + + "\x0fS3DownloadCreds\x12*\n" + + "\x11presigned_get_url\x18\x01 \x01(\tR\x0fpresignedGetUrl\x12\"\n" + + "\rexpires_at_ms\x18\x02 \x01(\x04R\vexpiresAtMs\x12\x15\n" + + "\x06s3_key\x18\x03 \x01(\tR\x05s3Key\"Y\n" + + "\vRetryPolicy\x12!\n" + + "\fmax_attempts\x18\x01 \x01(\rR\vmaxAttempts\x12'\n" + + "\x0fbackoff_seconds\x18\x02 \x03(\rR\x0ebackoffSeconds\"I\n" + + "\x11MaintenanceWindow\x12\x1b\n" + + "\tstart_utc\x18\x01 \x01(\tR\bstartUtc\x12\x17\n" + + "\aend_utc\x18\x02 \x01(\tR\x06endUtc\"\x8b\x01\n" + + "\x10AdvancedSettings\x12*\n" + + "\x11max_parallel_jobs\x18\x01 \x01(\rR\x0fmaxParallelJobs\x12\x1b\n" + + "\tlog_level\x18\x02 \x01(\tR\blogLevel\x12.\n" + + "\x13auto_update_enabled\x18\x03 \x01(\bR\x11autoUpdateEnabled*h\n" + + "\tJobStatus\x12\x1a\n" + + "\x16JOB_STATUS_UNSPECIFIED\x10\x00\x12\n" + + "\n" + + "\x06QUEUED\x10\x01\x12\v\n" + + "\aRUNNING\x10\x02\x12\v\n" + + "\aSUCCESS\x10\x03\x12\n" + + "\n" + + "\x06FAILED\x10\x04\x12\r\n" + + "\tCANCELLED\x10\x05*h\n" + + "\x06DbType\x12\x12\n" + + "\x0eDB_UNSPECIFIED\x10\x00\x12\x0e\n" + + "\n" + + "POSTGRESQL\x10\x01\x12\t\n" + + "\x05MYSQL\x10\x02\x12\v\n" + + "\aMARIADB\x10\x03\x12\v\n" + + "\aMONGODB\x10\x04\x12\t\n" + + "\x05REDIS\x10\x05\x12\n" + + "\n" + + "\x06SQLITE\x10\x06*?\n" + + "\x12ConnectionStrategy\x12\a\n" + + "\x03TCP\x10\x00\x12\x0f\n" + + "\vDOCKER_EXEC\x10\x01\x12\x0f\n" + + "\vUNIX_SOCKET\x10\x02*/\n" + + "\tCheckType\x12\b\n" + + "\x04HTTP\x10\x00\x12\t\n" + + "\x05HTTPS\x10\x01\x12\r\n" + + "\tTCP_CHECK\x10\x02*?\n" + + "\bLogLevel\x12\t\n" + + "\x05TRACE\x10\x00\x12\t\n" + + "\x05DEBUG\x10\x01\x12\b\n" + + "\x04INFO\x10\x02\x12\b\n" + + "\x04WARN\x10\x03\x12\t\n" + + "\x05ERROR\x10\x04B\xa5\x01\n" + + "\rcom.backup.v1B\vCommonProtoP\x01ZBgithub.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1\xa2\x02\x03BXX\xaa\x02\tBackup.V1\xca\x02\tBackup\\V1\xe2\x02\x15Backup\\V1\\GPBMetadata\xea\x02\n" + + "Backup::V1b\x06proto3" + +var ( + file_backupv1_common_proto_rawDescOnce sync.Once + file_backupv1_common_proto_rawDescData []byte +) + +func file_backupv1_common_proto_rawDescGZIP() []byte { + file_backupv1_common_proto_rawDescOnce.Do(func() { + file_backupv1_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_backupv1_common_proto_rawDesc), len(file_backupv1_common_proto_rawDesc))) + }) + return file_backupv1_common_proto_rawDescData +} + +var file_backupv1_common_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_backupv1_common_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_backupv1_common_proto_goTypes = []any{ + (JobStatus)(0), // 0: backup.v1.JobStatus + (DbType)(0), // 1: backup.v1.DbType + (ConnectionStrategy)(0), // 2: backup.v1.ConnectionStrategy + (CheckType)(0), // 3: backup.v1.CheckType + (LogLevel)(0), // 4: backup.v1.LogLevel + (*AgentMetrics)(nil), // 5: backup.v1.AgentMetrics + (*PortBinding)(nil), // 6: backup.v1.PortBinding + (*S3UploadCreds)(nil), // 7: backup.v1.S3UploadCreds + (*S3DownloadCreds)(nil), // 8: backup.v1.S3DownloadCreds + (*RetryPolicy)(nil), // 9: backup.v1.RetryPolicy + (*MaintenanceWindow)(nil), // 10: backup.v1.MaintenanceWindow + (*AdvancedSettings)(nil), // 11: backup.v1.AdvancedSettings +} +var file_backupv1_common_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_backupv1_common_proto_init() } +func file_backupv1_common_proto_init() { + if File_backupv1_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_backupv1_common_proto_rawDesc), len(file_backupv1_common_proto_rawDesc)), + NumEnums: 5, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_backupv1_common_proto_goTypes, + DependencyIndexes: file_backupv1_common_proto_depIdxs, + EnumInfos: file_backupv1_common_proto_enumTypes, + MessageInfos: file_backupv1_common_proto_msgTypes, + }.Build() + File_backupv1_common_proto = out.File + file_backupv1_common_proto_goTypes = nil + file_backupv1_common_proto_depIdxs = nil +} diff --git a/packages/proto/gen/go/backupv1/envelope.pb.go b/packages/proto/gen/go/backupv1/envelope.pb.go new file mode 100644 index 0000000..b156cd6 --- /dev/null +++ b/packages/proto/gen/go/backupv1/envelope.pb.go @@ -0,0 +1,497 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Envelope is the single top-level wrapper carried in every WSS binary frame +// in either direction. The `payload` oneof selects which concrete message is +// being transported; field numbers must remain stable forever. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: backupv1/envelope.proto + +package backupv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Envelope wraps every WSS message in both directions. +// See docs/07-api-contract.md §3. +type Envelope struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seq uint64 `protobuf:"varint,1,opt,name=seq,proto3" json:"seq,omitempty"` // monotonic, per-direction + TsMs uint64 `protobuf:"varint,2,opt,name=ts_ms,json=tsMs,proto3" json:"ts_ms,omitempty"` // unix milliseconds, set by sender + CorrelationId string `protobuf:"bytes,3,opt,name=correlation_id,json=correlationId,proto3" json:"correlation_id,omitempty"` // links request/response pairs (e.g. Ping/Ack) + // Types that are valid to be assigned to Payload: + // + // *Envelope_Register + // *Envelope_Heartbeat + // *Envelope_Discovery + // *Envelope_JobUpdate + // *Envelope_BackupCompleted + // *Envelope_HealthResult + // *Envelope_Log + // *Envelope_RestoreUpdate + // *Envelope_Ack + // *Envelope_RegisterAck + // *Envelope_ConfigUpdate + // *Envelope_RunBackup + // *Envelope_CancelJob + // *Envelope_RunHealthCheck + // *Envelope_SelfUpdate + // *Envelope_Ping + Payload isEnvelope_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Envelope) Reset() { + *x = Envelope{} + mi := &file_backupv1_envelope_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Envelope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Envelope) ProtoMessage() {} + +func (x *Envelope) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_envelope_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Envelope.ProtoReflect.Descriptor instead. +func (*Envelope) Descriptor() ([]byte, []int) { + return file_backupv1_envelope_proto_rawDescGZIP(), []int{0} +} + +func (x *Envelope) GetSeq() uint64 { + if x != nil { + return x.Seq + } + return 0 +} + +func (x *Envelope) GetTsMs() uint64 { + if x != nil { + return x.TsMs + } + return 0 +} + +func (x *Envelope) GetCorrelationId() string { + if x != nil { + return x.CorrelationId + } + return "" +} + +func (x *Envelope) GetPayload() isEnvelope_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *Envelope) GetRegister() *Register { + if x != nil { + if x, ok := x.Payload.(*Envelope_Register); ok { + return x.Register + } + } + return nil +} + +func (x *Envelope) GetHeartbeat() *Heartbeat { + if x != nil { + if x, ok := x.Payload.(*Envelope_Heartbeat); ok { + return x.Heartbeat + } + } + return nil +} + +func (x *Envelope) GetDiscovery() *DiscoveryReport { + if x != nil { + if x, ok := x.Payload.(*Envelope_Discovery); ok { + return x.Discovery + } + } + return nil +} + +func (x *Envelope) GetJobUpdate() *JobUpdate { + if x != nil { + if x, ok := x.Payload.(*Envelope_JobUpdate); ok { + return x.JobUpdate + } + } + return nil +} + +func (x *Envelope) GetBackupCompleted() *BackupCompleted { + if x != nil { + if x, ok := x.Payload.(*Envelope_BackupCompleted); ok { + return x.BackupCompleted + } + } + return nil +} + +func (x *Envelope) GetHealthResult() *HealthCheckResult { + if x != nil { + if x, ok := x.Payload.(*Envelope_HealthResult); ok { + return x.HealthResult + } + } + return nil +} + +func (x *Envelope) GetLog() *LogEvent { + if x != nil { + if x, ok := x.Payload.(*Envelope_Log); ok { + return x.Log + } + } + return nil +} + +func (x *Envelope) GetRestoreUpdate() *RestoreUpdate { + if x != nil { + if x, ok := x.Payload.(*Envelope_RestoreUpdate); ok { + return x.RestoreUpdate + } + } + return nil +} + +func (x *Envelope) GetAck() *Ack { + if x != nil { + if x, ok := x.Payload.(*Envelope_Ack); ok { + return x.Ack + } + } + return nil +} + +func (x *Envelope) GetRegisterAck() *RegisterAck { + if x != nil { + if x, ok := x.Payload.(*Envelope_RegisterAck); ok { + return x.RegisterAck + } + } + return nil +} + +func (x *Envelope) GetConfigUpdate() *ConfigUpdate { + if x != nil { + if x, ok := x.Payload.(*Envelope_ConfigUpdate); ok { + return x.ConfigUpdate + } + } + return nil +} + +func (x *Envelope) GetRunBackup() *RunBackup { + if x != nil { + if x, ok := x.Payload.(*Envelope_RunBackup); ok { + return x.RunBackup + } + } + return nil +} + +func (x *Envelope) GetCancelJob() *CancelJob { + if x != nil { + if x, ok := x.Payload.(*Envelope_CancelJob); ok { + return x.CancelJob + } + } + return nil +} + +func (x *Envelope) GetRunHealthCheck() *RunHealthCheck { + if x != nil { + if x, ok := x.Payload.(*Envelope_RunHealthCheck); ok { + return x.RunHealthCheck + } + } + return nil +} + +func (x *Envelope) GetSelfUpdate() *SelfUpdate { + if x != nil { + if x, ok := x.Payload.(*Envelope_SelfUpdate); ok { + return x.SelfUpdate + } + } + return nil +} + +func (x *Envelope) GetPing() *Ping { + if x != nil { + if x, ok := x.Payload.(*Envelope_Ping); ok { + return x.Ping + } + } + return nil +} + +type isEnvelope_Payload interface { + isEnvelope_Payload() +} + +type Envelope_Register struct { + // ---------------- agent -> server ---------------- + Register *Register `protobuf:"bytes,10,opt,name=register,proto3,oneof"` +} + +type Envelope_Heartbeat struct { + Heartbeat *Heartbeat `protobuf:"bytes,11,opt,name=heartbeat,proto3,oneof"` +} + +type Envelope_Discovery struct { + Discovery *DiscoveryReport `protobuf:"bytes,12,opt,name=discovery,proto3,oneof"` +} + +type Envelope_JobUpdate struct { + JobUpdate *JobUpdate `protobuf:"bytes,13,opt,name=job_update,json=jobUpdate,proto3,oneof"` +} + +type Envelope_BackupCompleted struct { + BackupCompleted *BackupCompleted `protobuf:"bytes,14,opt,name=backup_completed,json=backupCompleted,proto3,oneof"` +} + +type Envelope_HealthResult struct { + HealthResult *HealthCheckResult `protobuf:"bytes,15,opt,name=health_result,json=healthResult,proto3,oneof"` +} + +type Envelope_Log struct { + Log *LogEvent `protobuf:"bytes,16,opt,name=log,proto3,oneof"` +} + +type Envelope_RestoreUpdate struct { + RestoreUpdate *RestoreUpdate `protobuf:"bytes,17,opt,name=restore_update,json=restoreUpdate,proto3,oneof"` +} + +type Envelope_Ack struct { + Ack *Ack `protobuf:"bytes,18,opt,name=ack,proto3,oneof"` +} + +type Envelope_RegisterAck struct { + // ---------------- server -> agent ---------------- + RegisterAck *RegisterAck `protobuf:"bytes,50,opt,name=register_ack,json=registerAck,proto3,oneof"` +} + +type Envelope_ConfigUpdate struct { + ConfigUpdate *ConfigUpdate `protobuf:"bytes,51,opt,name=config_update,json=configUpdate,proto3,oneof"` +} + +type Envelope_RunBackup struct { + RunBackup *RunBackup `protobuf:"bytes,52,opt,name=run_backup,json=runBackup,proto3,oneof"` +} + +type Envelope_CancelJob struct { + CancelJob *CancelJob `protobuf:"bytes,53,opt,name=cancel_job,json=cancelJob,proto3,oneof"` +} + +type Envelope_RunHealthCheck struct { + RunHealthCheck *RunHealthCheck `protobuf:"bytes,54,opt,name=run_health_check,json=runHealthCheck,proto3,oneof"` +} + +type Envelope_SelfUpdate struct { + // 55 reserved: previously RunRestore (removed in MVP, see docs/07 §5). + SelfUpdate *SelfUpdate `protobuf:"bytes,56,opt,name=self_update,json=selfUpdate,proto3,oneof"` +} + +type Envelope_Ping struct { + Ping *Ping `protobuf:"bytes,57,opt,name=ping,proto3,oneof"` +} + +func (*Envelope_Register) isEnvelope_Payload() {} + +func (*Envelope_Heartbeat) isEnvelope_Payload() {} + +func (*Envelope_Discovery) isEnvelope_Payload() {} + +func (*Envelope_JobUpdate) isEnvelope_Payload() {} + +func (*Envelope_BackupCompleted) isEnvelope_Payload() {} + +func (*Envelope_HealthResult) isEnvelope_Payload() {} + +func (*Envelope_Log) isEnvelope_Payload() {} + +func (*Envelope_RestoreUpdate) isEnvelope_Payload() {} + +func (*Envelope_Ack) isEnvelope_Payload() {} + +func (*Envelope_RegisterAck) isEnvelope_Payload() {} + +func (*Envelope_ConfigUpdate) isEnvelope_Payload() {} + +func (*Envelope_RunBackup) isEnvelope_Payload() {} + +func (*Envelope_CancelJob) isEnvelope_Payload() {} + +func (*Envelope_RunHealthCheck) isEnvelope_Payload() {} + +func (*Envelope_SelfUpdate) isEnvelope_Payload() {} + +func (*Envelope_Ping) isEnvelope_Payload() {} + +var File_backupv1_envelope_proto protoreflect.FileDescriptor + +const file_backupv1_envelope_proto_rawDesc = "" + + "\n" + + "\x17backupv1/envelope.proto\x12\tbackup.v1\x1a\x1ebackupv1/agent_to_server.proto\x1a\x1ebackupv1/server_to_agent.proto\"\xf6\a\n" + + "\bEnvelope\x12\x10\n" + + "\x03seq\x18\x01 \x01(\x04R\x03seq\x12\x13\n" + + "\x05ts_ms\x18\x02 \x01(\x04R\x04tsMs\x12%\n" + + "\x0ecorrelation_id\x18\x03 \x01(\tR\rcorrelationId\x121\n" + + "\bregister\x18\n" + + " \x01(\v2\x13.backup.v1.RegisterH\x00R\bregister\x124\n" + + "\theartbeat\x18\v \x01(\v2\x14.backup.v1.HeartbeatH\x00R\theartbeat\x12:\n" + + "\tdiscovery\x18\f \x01(\v2\x1a.backup.v1.DiscoveryReportH\x00R\tdiscovery\x125\n" + + "\n" + + "job_update\x18\r \x01(\v2\x14.backup.v1.JobUpdateH\x00R\tjobUpdate\x12G\n" + + "\x10backup_completed\x18\x0e \x01(\v2\x1a.backup.v1.BackupCompletedH\x00R\x0fbackupCompleted\x12C\n" + + "\rhealth_result\x18\x0f \x01(\v2\x1c.backup.v1.HealthCheckResultH\x00R\fhealthResult\x12'\n" + + "\x03log\x18\x10 \x01(\v2\x13.backup.v1.LogEventH\x00R\x03log\x12A\n" + + "\x0erestore_update\x18\x11 \x01(\v2\x18.backup.v1.RestoreUpdateH\x00R\rrestoreUpdate\x12\"\n" + + "\x03ack\x18\x12 \x01(\v2\x0e.backup.v1.AckH\x00R\x03ack\x12;\n" + + "\fregister_ack\x182 \x01(\v2\x16.backup.v1.RegisterAckH\x00R\vregisterAck\x12>\n" + + "\rconfig_update\x183 \x01(\v2\x17.backup.v1.ConfigUpdateH\x00R\fconfigUpdate\x125\n" + + "\n" + + "run_backup\x184 \x01(\v2\x14.backup.v1.RunBackupH\x00R\trunBackup\x125\n" + + "\n" + + "cancel_job\x185 \x01(\v2\x14.backup.v1.CancelJobH\x00R\tcancelJob\x12E\n" + + "\x10run_health_check\x186 \x01(\v2\x19.backup.v1.RunHealthCheckH\x00R\x0erunHealthCheck\x128\n" + + "\vself_update\x188 \x01(\v2\x15.backup.v1.SelfUpdateH\x00R\n" + + "selfUpdate\x12%\n" + + "\x04ping\x189 \x01(\v2\x0f.backup.v1.PingH\x00R\x04pingB\t\n" + + "\apayloadJ\x04\b7\x108B\xa7\x01\n" + + "\rcom.backup.v1B\rEnvelopeProtoP\x01ZBgithub.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1\xa2\x02\x03BXX\xaa\x02\tBackup.V1\xca\x02\tBackup\\V1\xe2\x02\x15Backup\\V1\\GPBMetadata\xea\x02\n" + + "Backup::V1b\x06proto3" + +var ( + file_backupv1_envelope_proto_rawDescOnce sync.Once + file_backupv1_envelope_proto_rawDescData []byte +) + +func file_backupv1_envelope_proto_rawDescGZIP() []byte { + file_backupv1_envelope_proto_rawDescOnce.Do(func() { + file_backupv1_envelope_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_backupv1_envelope_proto_rawDesc), len(file_backupv1_envelope_proto_rawDesc))) + }) + return file_backupv1_envelope_proto_rawDescData +} + +var file_backupv1_envelope_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_backupv1_envelope_proto_goTypes = []any{ + (*Envelope)(nil), // 0: backup.v1.Envelope + (*Register)(nil), // 1: backup.v1.Register + (*Heartbeat)(nil), // 2: backup.v1.Heartbeat + (*DiscoveryReport)(nil), // 3: backup.v1.DiscoveryReport + (*JobUpdate)(nil), // 4: backup.v1.JobUpdate + (*BackupCompleted)(nil), // 5: backup.v1.BackupCompleted + (*HealthCheckResult)(nil), // 6: backup.v1.HealthCheckResult + (*LogEvent)(nil), // 7: backup.v1.LogEvent + (*RestoreUpdate)(nil), // 8: backup.v1.RestoreUpdate + (*Ack)(nil), // 9: backup.v1.Ack + (*RegisterAck)(nil), // 10: backup.v1.RegisterAck + (*ConfigUpdate)(nil), // 11: backup.v1.ConfigUpdate + (*RunBackup)(nil), // 12: backup.v1.RunBackup + (*CancelJob)(nil), // 13: backup.v1.CancelJob + (*RunHealthCheck)(nil), // 14: backup.v1.RunHealthCheck + (*SelfUpdate)(nil), // 15: backup.v1.SelfUpdate + (*Ping)(nil), // 16: backup.v1.Ping +} +var file_backupv1_envelope_proto_depIdxs = []int32{ + 1, // 0: backup.v1.Envelope.register:type_name -> backup.v1.Register + 2, // 1: backup.v1.Envelope.heartbeat:type_name -> backup.v1.Heartbeat + 3, // 2: backup.v1.Envelope.discovery:type_name -> backup.v1.DiscoveryReport + 4, // 3: backup.v1.Envelope.job_update:type_name -> backup.v1.JobUpdate + 5, // 4: backup.v1.Envelope.backup_completed:type_name -> backup.v1.BackupCompleted + 6, // 5: backup.v1.Envelope.health_result:type_name -> backup.v1.HealthCheckResult + 7, // 6: backup.v1.Envelope.log:type_name -> backup.v1.LogEvent + 8, // 7: backup.v1.Envelope.restore_update:type_name -> backup.v1.RestoreUpdate + 9, // 8: backup.v1.Envelope.ack:type_name -> backup.v1.Ack + 10, // 9: backup.v1.Envelope.register_ack:type_name -> backup.v1.RegisterAck + 11, // 10: backup.v1.Envelope.config_update:type_name -> backup.v1.ConfigUpdate + 12, // 11: backup.v1.Envelope.run_backup:type_name -> backup.v1.RunBackup + 13, // 12: backup.v1.Envelope.cancel_job:type_name -> backup.v1.CancelJob + 14, // 13: backup.v1.Envelope.run_health_check:type_name -> backup.v1.RunHealthCheck + 15, // 14: backup.v1.Envelope.self_update:type_name -> backup.v1.SelfUpdate + 16, // 15: backup.v1.Envelope.ping:type_name -> backup.v1.Ping + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name +} + +func init() { file_backupv1_envelope_proto_init() } +func file_backupv1_envelope_proto_init() { + if File_backupv1_envelope_proto != nil { + return + } + file_backupv1_agent_to_server_proto_init() + file_backupv1_server_to_agent_proto_init() + file_backupv1_envelope_proto_msgTypes[0].OneofWrappers = []any{ + (*Envelope_Register)(nil), + (*Envelope_Heartbeat)(nil), + (*Envelope_Discovery)(nil), + (*Envelope_JobUpdate)(nil), + (*Envelope_BackupCompleted)(nil), + (*Envelope_HealthResult)(nil), + (*Envelope_Log)(nil), + (*Envelope_RestoreUpdate)(nil), + (*Envelope_Ack)(nil), + (*Envelope_RegisterAck)(nil), + (*Envelope_ConfigUpdate)(nil), + (*Envelope_RunBackup)(nil), + (*Envelope_CancelJob)(nil), + (*Envelope_RunHealthCheck)(nil), + (*Envelope_SelfUpdate)(nil), + (*Envelope_Ping)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_backupv1_envelope_proto_rawDesc), len(file_backupv1_envelope_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_backupv1_envelope_proto_goTypes, + DependencyIndexes: file_backupv1_envelope_proto_depIdxs, + MessageInfos: file_backupv1_envelope_proto_msgTypes, + }.Build() + File_backupv1_envelope_proto = out.File + file_backupv1_envelope_proto_goTypes = nil + file_backupv1_envelope_proto_depIdxs = nil +} diff --git a/packages/proto/gen/go/backupv1/go.mod b/packages/proto/gen/go/backupv1/go.mod new file mode 100644 index 0000000..8bda5cb --- /dev/null +++ b/packages/proto/gen/go/backupv1/go.mod @@ -0,0 +1,5 @@ +module github.com/backupy/backupy/packages/proto/gen/go/backupv1 + +go 1.22 + +require google.golang.org/protobuf v1.34.2 diff --git a/packages/proto/gen/go/backupv1/server_to_agent.pb.go b/packages/proto/gen/go/backupv1/server_to_agent.pb.go new file mode 100644 index 0000000..d62af23 --- /dev/null +++ b/packages/proto/gen/go/backupv1/server_to_agent.pb.go @@ -0,0 +1,1063 @@ +// Backupy protobuf v1 — see docs/07-api-contract.md +// +// Messages flowing from the server to the agent over the long-lived WSS +// connection. Each is wrapped in an Envelope (see envelope.proto). + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: backupv1/server_to_agent.proto + +package backupv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// RegisterAck is the reply to Register. Carries an AgentConfig snapshot so the +// agent can begin work without an extra round-trip. +// See docs/07-api-contract.md §5 (RegisterAck). +type RegisterAck struct { + state protoimpl.MessageState `protogen:"open.v1"` + SessionId string `protobuf:"bytes,1,opt,name=session_id,json=sessionId,proto3" json:"session_id,omitempty"` + Config *AgentConfig `protobuf:"bytes,2,opt,name=config,proto3" json:"config,omitempty"` + HeartbeatIntervalSec uint32 `protobuf:"varint,3,opt,name=heartbeat_interval_sec,json=heartbeatIntervalSec,proto3" json:"heartbeat_interval_sec,omitempty"` + ServerTimeIso string `protobuf:"bytes,4,opt,name=server_time_iso,json=serverTimeIso,proto3" json:"server_time_iso,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RegisterAck) Reset() { + *x = RegisterAck{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RegisterAck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterAck) ProtoMessage() {} + +func (x *RegisterAck) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterAck.ProtoReflect.Descriptor instead. +func (*RegisterAck) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{0} +} + +func (x *RegisterAck) GetSessionId() string { + if x != nil { + return x.SessionId + } + return "" +} + +func (x *RegisterAck) GetConfig() *AgentConfig { + if x != nil { + return x.Config + } + return nil +} + +func (x *RegisterAck) GetHeartbeatIntervalSec() uint32 { + if x != nil { + return x.HeartbeatIntervalSec + } + return 0 +} + +func (x *RegisterAck) GetServerTimeIso() string { + if x != nil { + return x.ServerTimeIso + } + return "" +} + +// AgentConfig is the full configuration snapshot for an agent. +// See docs/07-api-contract.md §5 (AgentConfig). +type AgentConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version uint64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Targets []*Target `protobuf:"bytes,2,rep,name=targets,proto3" json:"targets,omitempty"` + Jobs []*BackupJobSpec `protobuf:"bytes,3,rep,name=jobs,proto3" json:"jobs,omitempty"` + HealthChecks []*HealthCheckSpec `protobuf:"bytes,4,rep,name=health_checks,json=healthChecks,proto3" json:"health_checks,omitempty"` + Maintenance *MaintenanceWindow `protobuf:"bytes,5,opt,name=maintenance,proto3" json:"maintenance,omitempty"` + Advanced *AdvancedSettings `protobuf:"bytes,6,opt,name=advanced,proto3" json:"advanced,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AgentConfig) Reset() { + *x = AgentConfig{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AgentConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AgentConfig) ProtoMessage() {} + +func (x *AgentConfig) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AgentConfig.ProtoReflect.Descriptor instead. +func (*AgentConfig) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{1} +} + +func (x *AgentConfig) GetVersion() uint64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *AgentConfig) GetTargets() []*Target { + if x != nil { + return x.Targets + } + return nil +} + +func (x *AgentConfig) GetJobs() []*BackupJobSpec { + if x != nil { + return x.Jobs + } + return nil +} + +func (x *AgentConfig) GetHealthChecks() []*HealthCheckSpec { + if x != nil { + return x.HealthChecks + } + return nil +} + +func (x *AgentConfig) GetMaintenance() *MaintenanceWindow { + if x != nil { + return x.Maintenance + } + return nil +} + +func (x *AgentConfig) GetAdvanced() *AdvancedSettings { + if x != nil { + return x.Advanced + } + return nil +} + +// Target is a single backup source (database) managed by this agent. +// See docs/07-api-contract.md §5 (Target). +type Target struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Type DbType `protobuf:"varint,2,opt,name=type,proto3,enum=backup.v1.DbType" json:"type,omitempty"` + DisplayName string `protobuf:"bytes,3,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"` + Connection *ConnectionConfig `protobuf:"bytes,4,opt,name=connection,proto3" json:"connection,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Target) Reset() { + *x = Target{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Target) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Target) ProtoMessage() {} + +func (x *Target) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Target.ProtoReflect.Descriptor instead. +func (*Target) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{2} +} + +func (x *Target) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Target) GetType() DbType { + if x != nil { + return x.Type + } + return DbType_DB_UNSPECIFIED +} + +func (x *Target) GetDisplayName() string { + if x != nil { + return x.DisplayName + } + return "" +} + +func (x *Target) GetConnection() *ConnectionConfig { + if x != nil { + return x.Connection + } + return nil +} + +// ConnectionConfig describes how the agent connects to a Target's database. +// See docs/07-api-contract.md §5 (ConnectionConfig). +type ConnectionConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + Port uint32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + Database string `protobuf:"bytes,3,opt,name=database,proto3" json:"database,omitempty"` + Username string `protobuf:"bytes,4,opt,name=username,proto3" json:"username,omitempty"` + PasswordSecretRef string `protobuf:"bytes,5,opt,name=password_secret_ref,json=passwordSecretRef,proto3" json:"password_secret_ref,omitempty"` // reference to encrypted secret in vault + ContainerId string `protobuf:"bytes,6,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"` // populated for DOCKER_EXEC strategy + Strategy ConnectionStrategy `protobuf:"varint,7,opt,name=strategy,proto3,enum=backup.v1.ConnectionStrategy" json:"strategy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConnectionConfig) Reset() { + *x = ConnectionConfig{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConnectionConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectionConfig) ProtoMessage() {} + +func (x *ConnectionConfig) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectionConfig.ProtoReflect.Descriptor instead. +func (*ConnectionConfig) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *ConnectionConfig) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *ConnectionConfig) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ConnectionConfig) GetDatabase() string { + if x != nil { + return x.Database + } + return "" +} + +func (x *ConnectionConfig) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *ConnectionConfig) GetPasswordSecretRef() string { + if x != nil { + return x.PasswordSecretRef + } + return "" +} + +func (x *ConnectionConfig) GetContainerId() string { + if x != nil { + return x.ContainerId + } + return "" +} + +func (x *ConnectionConfig) GetStrategy() ConnectionStrategy { + if x != nil { + return x.Strategy + } + return ConnectionStrategy_TCP +} + +// BackupJobSpec defines a scheduled backup job for one Target. +// See docs/07-api-contract.md §5 (BackupJobSpec). +type BackupJobSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + TargetId string `protobuf:"bytes,2,opt,name=target_id,json=targetId,proto3" json:"target_id,omitempty"` + Cron string `protobuf:"bytes,3,opt,name=cron,proto3" json:"cron,omitempty"` // cron expression in UTC + RetentionDays uint32 `protobuf:"varint,4,opt,name=retention_days,json=retentionDays,proto3" json:"retention_days,omitempty"` + Compression string `protobuf:"bytes,5,opt,name=compression,proto3" json:"compression,omitempty"` // "zstd" | "gzip" | "none" + EncryptionEnabled bool `protobuf:"varint,6,opt,name=encryption_enabled,json=encryptionEnabled,proto3" json:"encryption_enabled,omitempty"` + PreHooks []string `protobuf:"bytes,7,rep,name=pre_hooks,json=preHooks,proto3" json:"pre_hooks,omitempty"` + PostHooks []string `protobuf:"bytes,8,rep,name=post_hooks,json=postHooks,proto3" json:"post_hooks,omitempty"` + Retry *RetryPolicy `protobuf:"bytes,9,opt,name=retry,proto3" json:"retry,omitempty"` + TimeoutSec uint32 `protobuf:"varint,10,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupJobSpec) Reset() { + *x = BackupJobSpec{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupJobSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupJobSpec) ProtoMessage() {} + +func (x *BackupJobSpec) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupJobSpec.ProtoReflect.Descriptor instead. +func (*BackupJobSpec) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{4} +} + +func (x *BackupJobSpec) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *BackupJobSpec) GetTargetId() string { + if x != nil { + return x.TargetId + } + return "" +} + +func (x *BackupJobSpec) GetCron() string { + if x != nil { + return x.Cron + } + return "" +} + +func (x *BackupJobSpec) GetRetentionDays() uint32 { + if x != nil { + return x.RetentionDays + } + return 0 +} + +func (x *BackupJobSpec) GetCompression() string { + if x != nil { + return x.Compression + } + return "" +} + +func (x *BackupJobSpec) GetEncryptionEnabled() bool { + if x != nil { + return x.EncryptionEnabled + } + return false +} + +func (x *BackupJobSpec) GetPreHooks() []string { + if x != nil { + return x.PreHooks + } + return nil +} + +func (x *BackupJobSpec) GetPostHooks() []string { + if x != nil { + return x.PostHooks + } + return nil +} + +func (x *BackupJobSpec) GetRetry() *RetryPolicy { + if x != nil { + return x.Retry + } + return nil +} + +func (x *BackupJobSpec) GetTimeoutSec() uint32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +// HealthCheckSpec defines a single health-check probe. +// See docs/07-api-contract.md §5 (HealthCheckSpec). +type HealthCheckSpec struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Type CheckType `protobuf:"varint,3,opt,name=type,proto3,enum=backup.v1.CheckType" json:"type,omitempty"` + Target string `protobuf:"bytes,4,opt,name=target,proto3" json:"target,omitempty"` // URL or host:port + IntervalSec uint32 `protobuf:"varint,5,opt,name=interval_sec,json=intervalSec,proto3" json:"interval_sec,omitempty"` + TimeoutSec uint32 `protobuf:"varint,6,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` + ExpectedStatus uint32 `protobuf:"varint,7,opt,name=expected_status,json=expectedStatus,proto3" json:"expected_status,omitempty"` // expected HTTP status (HTTP/HTTPS only) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthCheckSpec) Reset() { + *x = HealthCheckSpec{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthCheckSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthCheckSpec) ProtoMessage() {} + +func (x *HealthCheckSpec) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthCheckSpec.ProtoReflect.Descriptor instead. +func (*HealthCheckSpec) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{5} +} + +func (x *HealthCheckSpec) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *HealthCheckSpec) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *HealthCheckSpec) GetType() CheckType { + if x != nil { + return x.Type + } + return CheckType_HTTP +} + +func (x *HealthCheckSpec) GetTarget() string { + if x != nil { + return x.Target + } + return "" +} + +func (x *HealthCheckSpec) GetIntervalSec() uint32 { + if x != nil { + return x.IntervalSec + } + return 0 +} + +func (x *HealthCheckSpec) GetTimeoutSec() uint32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +func (x *HealthCheckSpec) GetExpectedStatus() uint32 { + if x != nil { + return x.ExpectedStatus + } + return 0 +} + +// ConfigUpdate replaces the agent's current AgentConfig. +// See docs/07-api-contract.md §5. +type ConfigUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Config *AgentConfig `protobuf:"bytes,1,opt,name=config,proto3" json:"config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConfigUpdate) Reset() { + *x = ConfigUpdate{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConfigUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConfigUpdate) ProtoMessage() {} + +func (x *ConfigUpdate) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConfigUpdate.ProtoReflect.Descriptor instead. +func (*ConfigUpdate) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{6} +} + +func (x *ConfigUpdate) GetConfig() *AgentConfig { + if x != nil { + return x.Config + } + return nil +} + +// RunBackup tells the agent to execute one backup run for the named job. +// Server pre-issues S3 upload credentials and a KMS-wrapped DEK. +// See docs/07-api-contract.md §5 (RunBackup). +type RunBackup struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + RunId string `protobuf:"bytes,2,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ManualTrigger bool `protobuf:"varint,3,opt,name=manual_trigger,json=manualTrigger,proto3" json:"manual_trigger,omitempty"` + UploadCreds *S3UploadCreds `protobuf:"bytes,4,opt,name=upload_creds,json=uploadCreds,proto3" json:"upload_creds,omitempty"` + EncryptedDek []byte `protobuf:"bytes,5,opt,name=encrypted_dek,json=encryptedDek,proto3" json:"encrypted_dek,omitempty"` // DEK wrapped by KMS + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunBackup) Reset() { + *x = RunBackup{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunBackup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunBackup) ProtoMessage() {} + +func (x *RunBackup) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunBackup.ProtoReflect.Descriptor instead. +func (*RunBackup) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{7} +} + +func (x *RunBackup) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *RunBackup) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RunBackup) GetManualTrigger() bool { + if x != nil { + return x.ManualTrigger + } + return false +} + +func (x *RunBackup) GetUploadCreds() *S3UploadCreds { + if x != nil { + return x.UploadCreds + } + return nil +} + +func (x *RunBackup) GetEncryptedDek() []byte { + if x != nil { + return x.EncryptedDek + } + return nil +} + +// CancelJob requests cancellation of an in-flight job. +// See docs/07-api-contract.md §5 (CancelJob). +type CancelJob struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CancelJob) Reset() { + *x = CancelJob{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CancelJob) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelJob) ProtoMessage() {} + +func (x *CancelJob) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelJob.ProtoReflect.Descriptor instead. +func (*CancelJob) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{8} +} + +func (x *CancelJob) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +// RunHealthCheck triggers an immediate, out-of-schedule health-check probe. +// See docs/07-api-contract.md §5 (RunHealthCheck). +type RunHealthCheck struct { + state protoimpl.MessageState `protogen:"open.v1"` + CheckId string `protobuf:"bytes,1,opt,name=check_id,json=checkId,proto3" json:"check_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunHealthCheck) Reset() { + *x = RunHealthCheck{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunHealthCheck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunHealthCheck) ProtoMessage() {} + +func (x *RunHealthCheck) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunHealthCheck.ProtoReflect.Descriptor instead. +func (*RunHealthCheck) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{9} +} + +func (x *RunHealthCheck) GetCheckId() string { + if x != nil { + return x.CheckId + } + return "" +} + +// SelfUpdate asks the agent to upgrade its binary to target_version. +// cosign_signature lets the agent verify the binary before swapping it in. +// See docs/07-api-contract.md §5 (SelfUpdate). +type SelfUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + TargetVersion string `protobuf:"bytes,1,opt,name=target_version,json=targetVersion,proto3" json:"target_version,omitempty"` + BinaryUrl string `protobuf:"bytes,2,opt,name=binary_url,json=binaryUrl,proto3" json:"binary_url,omitempty"` + Sha256 string `protobuf:"bytes,3,opt,name=sha256,proto3" json:"sha256,omitempty"` + CosignSignature []byte `protobuf:"bytes,4,opt,name=cosign_signature,json=cosignSignature,proto3" json:"cosign_signature,omitempty"` + Force bool `protobuf:"varint,5,opt,name=force,proto3" json:"force,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SelfUpdate) Reset() { + *x = SelfUpdate{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SelfUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SelfUpdate) ProtoMessage() {} + +func (x *SelfUpdate) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SelfUpdate.ProtoReflect.Descriptor instead. +func (*SelfUpdate) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *SelfUpdate) GetTargetVersion() string { + if x != nil { + return x.TargetVersion + } + return "" +} + +func (x *SelfUpdate) GetBinaryUrl() string { + if x != nil { + return x.BinaryUrl + } + return "" +} + +func (x *SelfUpdate) GetSha256() string { + if x != nil { + return x.Sha256 + } + return "" +} + +func (x *SelfUpdate) GetCosignSignature() []byte { + if x != nil { + return x.CosignSignature + } + return nil +} + +func (x *SelfUpdate) GetForce() bool { + if x != nil { + return x.Force + } + return false +} + +// Ping is an application-level keepalive complementing WS-level pings. +// The agent replies with an Ack carrying the same correlation_id. +type Ping struct { + state protoimpl.MessageState `protogen:"open.v1"` + TsMs uint64 `protobuf:"varint,1,opt,name=ts_ms,json=tsMs,proto3" json:"ts_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ping) Reset() { + *x = Ping{} + mi := &file_backupv1_server_to_agent_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ping) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ping) ProtoMessage() {} + +func (x *Ping) ProtoReflect() protoreflect.Message { + mi := &file_backupv1_server_to_agent_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ping.ProtoReflect.Descriptor instead. +func (*Ping) Descriptor() ([]byte, []int) { + return file_backupv1_server_to_agent_proto_rawDescGZIP(), []int{11} +} + +func (x *Ping) GetTsMs() uint64 { + if x != nil { + return x.TsMs + } + return 0 +} + +var File_backupv1_server_to_agent_proto protoreflect.FileDescriptor + +const file_backupv1_server_to_agent_proto_rawDesc = "" + + "\n" + + "\x1ebackupv1/server_to_agent.proto\x12\tbackup.v1\x1a\x15backupv1/common.proto\"\xba\x01\n" + + "\vRegisterAck\x12\x1d\n" + + "\n" + + "session_id\x18\x01 \x01(\tR\tsessionId\x12.\n" + + "\x06config\x18\x02 \x01(\v2\x16.backup.v1.AgentConfigR\x06config\x124\n" + + "\x16heartbeat_interval_sec\x18\x03 \x01(\rR\x14heartbeatIntervalSec\x12&\n" + + "\x0fserver_time_iso\x18\x04 \x01(\tR\rserverTimeIso\"\xbc\x02\n" + + "\vAgentConfig\x12\x18\n" + + "\aversion\x18\x01 \x01(\x04R\aversion\x12+\n" + + "\atargets\x18\x02 \x03(\v2\x11.backup.v1.TargetR\atargets\x12,\n" + + "\x04jobs\x18\x03 \x03(\v2\x18.backup.v1.BackupJobSpecR\x04jobs\x12?\n" + + "\rhealth_checks\x18\x04 \x03(\v2\x1a.backup.v1.HealthCheckSpecR\fhealthChecks\x12>\n" + + "\vmaintenance\x18\x05 \x01(\v2\x1c.backup.v1.MaintenanceWindowR\vmaintenance\x127\n" + + "\badvanced\x18\x06 \x01(\v2\x1b.backup.v1.AdvancedSettingsR\badvanced\"\x9f\x01\n" + + "\x06Target\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12%\n" + + "\x04type\x18\x02 \x01(\x0e2\x11.backup.v1.DbTypeR\x04type\x12!\n" + + "\fdisplay_name\x18\x03 \x01(\tR\vdisplayName\x12;\n" + + "\n" + + "connection\x18\x04 \x01(\v2\x1b.backup.v1.ConnectionConfigR\n" + + "connection\"\x80\x02\n" + + "\x10ConnectionConfig\x12\x12\n" + + "\x04host\x18\x01 \x01(\tR\x04host\x12\x12\n" + + "\x04port\x18\x02 \x01(\rR\x04port\x12\x1a\n" + + "\bdatabase\x18\x03 \x01(\tR\bdatabase\x12\x1a\n" + + "\busername\x18\x04 \x01(\tR\busername\x12.\n" + + "\x13password_secret_ref\x18\x05 \x01(\tR\x11passwordSecretRef\x12!\n" + + "\fcontainer_id\x18\x06 \x01(\tR\vcontainerId\x129\n" + + "\bstrategy\x18\a \x01(\x0e2\x1d.backup.v1.ConnectionStrategyR\bstrategy\"\xd3\x02\n" + + "\rBackupJobSpec\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1b\n" + + "\ttarget_id\x18\x02 \x01(\tR\btargetId\x12\x12\n" + + "\x04cron\x18\x03 \x01(\tR\x04cron\x12%\n" + + "\x0eretention_days\x18\x04 \x01(\rR\rretentionDays\x12 \n" + + "\vcompression\x18\x05 \x01(\tR\vcompression\x12-\n" + + "\x12encryption_enabled\x18\x06 \x01(\bR\x11encryptionEnabled\x12\x1b\n" + + "\tpre_hooks\x18\a \x03(\tR\bpreHooks\x12\x1d\n" + + "\n" + + "post_hooks\x18\b \x03(\tR\tpostHooks\x12,\n" + + "\x05retry\x18\t \x01(\v2\x16.backup.v1.RetryPolicyR\x05retry\x12\x1f\n" + + "\vtimeout_sec\x18\n" + + " \x01(\rR\n" + + "timeoutSec\"\xe4\x01\n" + + "\x0fHealthCheckSpec\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12(\n" + + "\x04type\x18\x03 \x01(\x0e2\x14.backup.v1.CheckTypeR\x04type\x12\x16\n" + + "\x06target\x18\x04 \x01(\tR\x06target\x12!\n" + + "\finterval_sec\x18\x05 \x01(\rR\vintervalSec\x12\x1f\n" + + "\vtimeout_sec\x18\x06 \x01(\rR\n" + + "timeoutSec\x12'\n" + + "\x0fexpected_status\x18\a \x01(\rR\x0eexpectedStatus\">\n" + + "\fConfigUpdate\x12.\n" + + "\x06config\x18\x01 \x01(\v2\x16.backup.v1.AgentConfigR\x06config\"\xc2\x01\n" + + "\tRunBackup\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x15\n" + + "\x06run_id\x18\x02 \x01(\tR\x05runId\x12%\n" + + "\x0emanual_trigger\x18\x03 \x01(\bR\rmanualTrigger\x12;\n" + + "\fupload_creds\x18\x04 \x01(\v2\x18.backup.v1.S3UploadCredsR\vuploadCreds\x12#\n" + + "\rencrypted_dek\x18\x05 \x01(\fR\fencryptedDek\"\"\n" + + "\tCancelJob\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\"+\n" + + "\x0eRunHealthCheck\x12\x19\n" + + "\bcheck_id\x18\x01 \x01(\tR\acheckId\"\xab\x01\n" + + "\n" + + "SelfUpdate\x12%\n" + + "\x0etarget_version\x18\x01 \x01(\tR\rtargetVersion\x12\x1d\n" + + "\n" + + "binary_url\x18\x02 \x01(\tR\tbinaryUrl\x12\x16\n" + + "\x06sha256\x18\x03 \x01(\tR\x06sha256\x12)\n" + + "\x10cosign_signature\x18\x04 \x01(\fR\x0fcosignSignature\x12\x14\n" + + "\x05force\x18\x05 \x01(\bR\x05force\"\x1b\n" + + "\x04Ping\x12\x13\n" + + "\x05ts_ms\x18\x01 \x01(\x04R\x04tsMsB\xac\x01\n" + + "\rcom.backup.v1B\x12ServerToAgentProtoP\x01ZBgithub.com/backupy/backupy/packages/proto/gen/go/backupv1;backupv1\xa2\x02\x03BXX\xaa\x02\tBackup.V1\xca\x02\tBackup\\V1\xe2\x02\x15Backup\\V1\\GPBMetadata\xea\x02\n" + + "Backup::V1b\x06proto3" + +var ( + file_backupv1_server_to_agent_proto_rawDescOnce sync.Once + file_backupv1_server_to_agent_proto_rawDescData []byte +) + +func file_backupv1_server_to_agent_proto_rawDescGZIP() []byte { + file_backupv1_server_to_agent_proto_rawDescOnce.Do(func() { + file_backupv1_server_to_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_backupv1_server_to_agent_proto_rawDesc), len(file_backupv1_server_to_agent_proto_rawDesc))) + }) + return file_backupv1_server_to_agent_proto_rawDescData +} + +var file_backupv1_server_to_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_backupv1_server_to_agent_proto_goTypes = []any{ + (*RegisterAck)(nil), // 0: backup.v1.RegisterAck + (*AgentConfig)(nil), // 1: backup.v1.AgentConfig + (*Target)(nil), // 2: backup.v1.Target + (*ConnectionConfig)(nil), // 3: backup.v1.ConnectionConfig + (*BackupJobSpec)(nil), // 4: backup.v1.BackupJobSpec + (*HealthCheckSpec)(nil), // 5: backup.v1.HealthCheckSpec + (*ConfigUpdate)(nil), // 6: backup.v1.ConfigUpdate + (*RunBackup)(nil), // 7: backup.v1.RunBackup + (*CancelJob)(nil), // 8: backup.v1.CancelJob + (*RunHealthCheck)(nil), // 9: backup.v1.RunHealthCheck + (*SelfUpdate)(nil), // 10: backup.v1.SelfUpdate + (*Ping)(nil), // 11: backup.v1.Ping + (*MaintenanceWindow)(nil), // 12: backup.v1.MaintenanceWindow + (*AdvancedSettings)(nil), // 13: backup.v1.AdvancedSettings + (DbType)(0), // 14: backup.v1.DbType + (ConnectionStrategy)(0), // 15: backup.v1.ConnectionStrategy + (*RetryPolicy)(nil), // 16: backup.v1.RetryPolicy + (CheckType)(0), // 17: backup.v1.CheckType + (*S3UploadCreds)(nil), // 18: backup.v1.S3UploadCreds +} +var file_backupv1_server_to_agent_proto_depIdxs = []int32{ + 1, // 0: backup.v1.RegisterAck.config:type_name -> backup.v1.AgentConfig + 2, // 1: backup.v1.AgentConfig.targets:type_name -> backup.v1.Target + 4, // 2: backup.v1.AgentConfig.jobs:type_name -> backup.v1.BackupJobSpec + 5, // 3: backup.v1.AgentConfig.health_checks:type_name -> backup.v1.HealthCheckSpec + 12, // 4: backup.v1.AgentConfig.maintenance:type_name -> backup.v1.MaintenanceWindow + 13, // 5: backup.v1.AgentConfig.advanced:type_name -> backup.v1.AdvancedSettings + 14, // 6: backup.v1.Target.type:type_name -> backup.v1.DbType + 3, // 7: backup.v1.Target.connection:type_name -> backup.v1.ConnectionConfig + 15, // 8: backup.v1.ConnectionConfig.strategy:type_name -> backup.v1.ConnectionStrategy + 16, // 9: backup.v1.BackupJobSpec.retry:type_name -> backup.v1.RetryPolicy + 17, // 10: backup.v1.HealthCheckSpec.type:type_name -> backup.v1.CheckType + 1, // 11: backup.v1.ConfigUpdate.config:type_name -> backup.v1.AgentConfig + 18, // 12: backup.v1.RunBackup.upload_creds:type_name -> backup.v1.S3UploadCreds + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_backupv1_server_to_agent_proto_init() } +func file_backupv1_server_to_agent_proto_init() { + if File_backupv1_server_to_agent_proto != nil { + return + } + file_backupv1_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_backupv1_server_to_agent_proto_rawDesc), len(file_backupv1_server_to_agent_proto_rawDesc)), + NumEnums: 0, + NumMessages: 12, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_backupv1_server_to_agent_proto_goTypes, + DependencyIndexes: file_backupv1_server_to_agent_proto_depIdxs, + MessageInfos: file_backupv1_server_to_agent_proto_msgTypes, + }.Build() + File_backupv1_server_to_agent_proto = out.File + file_backupv1_server_to_agent_proto_goTypes = nil + file_backupv1_server_to_agent_proto_depIdxs = nil +}