mirror of
https://github.com/TronoSfera/backupy-agent.git
synced 2026-05-18 18:13:30 +03:00
Compare commits
7 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ff8882d864 | ||
|
|
6fe4d9165d | ||
|
|
3cfac4daca | ||
|
|
6a56577dab | ||
|
|
f9160a7686 | ||
|
|
f2a203d6cc | ||
|
|
55571c7c4c |
17 changed files with 247 additions and 71 deletions
|
|
@ -17,7 +17,7 @@ Open-source backup agent for the [Backupy](https://backupy.tronosfera.ru) backup
|
||||||
```yaml
|
```yaml
|
||||||
services:
|
services:
|
||||||
backupy-agent:
|
backupy-agent:
|
||||||
image: ghcr.io/tronosfera/backupy-agent:v0.1.0
|
image: ghcr.io/tronosfera/backupy-agent:0.1.0
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
BACKUPY_SERVER_URL: wss://backupy.tronosfera.ru/agents/connect
|
BACKUPY_SERVER_URL: wss://backupy.tronosfera.ru/agents/connect
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,7 @@ RUN apk add --no-cache \
|
||||||
mariadb-connector-c \
|
mariadb-connector-c \
|
||||||
openssl \
|
openssl \
|
||||||
sqlite-libs \
|
sqlite-libs \
|
||||||
|
readline \
|
||||||
&& addgroup -S backupy -g 1000 \
|
&& addgroup -S backupy -g 1000 \
|
||||||
&& adduser -S backupy -G backupy -u 1000
|
&& adduser -S backupy -G backupy -u 1000
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,8 @@ services:
|
||||||
backup-agent:
|
backup-agent:
|
||||||
image: backupy/agent:1
|
image: backupy/agent:1
|
||||||
environment:
|
environment:
|
||||||
BACKUP_SERVER_URL: https://api.backupy.ru
|
BACKUPY_SERVER_URL: https://api.backupy.ru
|
||||||
BACKUP_AGENT_KEY: ${BACKUP_KEY}
|
BACKUPY_AGENT_KEY: ${BACKUP_KEY}
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
- backup-agent-state:/var/lib/backup-agent
|
- backup-agent-state:/var/lib/backup-agent
|
||||||
|
|
@ -46,12 +46,12 @@ volumes:
|
||||||
|
|
||||||
| Var | Required | Default | Notes |
|
| Var | Required | Default | Notes |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `BACKUP_SERVER_URL` | yes | `https://api.backupy.ru` | Must be `https://` (override with `BACKUP_DEV_ALLOW_INSECURE=true` for local dev). |
|
| `BACKUPY_SERVER_URL` | yes | `https://api.backupy.ru` | Must be `https://` (override with `BACKUPY_DEV_ALLOW_INSECURE=true` for local dev). |
|
||||||
| `BACKUP_AGENT_KEY` | yes | — | Format `bkpy_(live\|test)_<32 alnum>`. Never logged. |
|
| `BACKUPY_AGENT_KEY` | yes | — | Format `bkpy_(live\|test)_<32 alnum>`. Never logged. |
|
||||||
| `BACKUP_STATE_DIR` | no | `/var/lib/backup-agent` | Volume-mounted. Must be writable by uid 65532 (distroless `nonroot`). |
|
| `BACKUPY_STATE_DIR` | no | `/var/lib/backup-agent` | Volume-mounted. Must be writable by uid 65532 (distroless `nonroot`). |
|
||||||
| `BACKUP_LOG_LEVEL` | no | `info` | `trace`/`debug`/`info`/`warn`/`error`. |
|
| `BACKUPY_LOG_LEVEL` | no | `info` | `trace`/`debug`/`info`/`warn`/`error`. |
|
||||||
| `BACKUP_DOCKER_SOCKET` | no | `/var/run/docker.sock` | Mounted read-only. |
|
| `BACKUPY_DOCKER_SOCKET` | no | `/var/run/docker.sock` | Mounted read-only. |
|
||||||
| `BACKUP_DEV_ALLOW_INSECURE` | no | `false` | Allows `http://` server URL — dev only. |
|
| `BACKUPY_DEV_ALLOW_INSECURE` | no | `false` | Allows `http://` server URL — dev only. |
|
||||||
|
|
||||||
Everything else (targets, schedules, retention, S3 creds, hooks) comes from
|
Everything else (targets, schedules, retention, S3 creds, hooks) comes from
|
||||||
the server via `ConfigUpdate` after registration.
|
the server via `ConfigUpdate` after registration.
|
||||||
|
|
@ -127,7 +127,7 @@ Packages that require generated proto code to compile:
|
||||||
## Security notes
|
## Security notes
|
||||||
|
|
||||||
- TLS 1.3 to all server endpoints (enforced by `coder/websocket` defaults).
|
- TLS 1.3 to all server endpoints (enforced by `coder/websocket` defaults).
|
||||||
- `BACKUP_AGENT_KEY` is never logged (`slog` ReplaceAttr redacts known keys
|
- `BACKUPY_AGENT_KEY` is never logged (`slog` ReplaceAttr redacts known keys
|
||||||
defensively; the value is also `json:"-"` in `Config`).
|
defensively; the value is also `json:"-"` in `Config`).
|
||||||
- State at rest is AES-256-GCM keyed by HKDF-SHA256 of the agent key.
|
- State at rest is AES-256-GCM keyed by HKDF-SHA256 of the agent key.
|
||||||
- Docker socket is mounted read-only.
|
- Docker socket is mounted read-only.
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ import (
|
||||||
//
|
//
|
||||||
// Health criteria:
|
// Health criteria:
|
||||||
//
|
//
|
||||||
// 1. Required env vars are set (BACKUP_AGENT_KEY / BACKUP_SERVER_URL).
|
// 1. Required env vars are set (BACKUPY_AGENT_KEY / BACKUPY_SERVER_URL).
|
||||||
// 2. The state.db file can be opened (validates encryption key + on-disk
|
// 2. The state.db file can be opened (validates encryption key + on-disk
|
||||||
// integrity).
|
// integrity).
|
||||||
//
|
//
|
||||||
|
|
|
||||||
|
|
@ -20,29 +20,29 @@ import (
|
||||||
"github.com/caarlos0/env/v11"
|
"github.com/caarlos0/env/v11"
|
||||||
)
|
)
|
||||||
|
|
||||||
// agentKeyPattern enforces the documented BACKUP_AGENT_KEY format
|
// agentKeyPattern enforces the documented BACKUPY_AGENT_KEY format
|
||||||
// `bkpy_(live|test)_<32 base62 chars>`. The server issues keys in
|
// `bkpy_(live|test)_<32 base62 chars>`. The server issues keys in
|
||||||
// this exact shape — see docs/03-agent-spec.md and server task A-09.
|
// this exact shape — see docs/03-agent-spec.md and server task A-09.
|
||||||
var agentKeyPattern = regexp.MustCompile(`^bkpy_(live|test)_[A-Za-z0-9]{32}$`)
|
var agentKeyPattern = regexp.MustCompile(`^(?:bkpy_(?:live|test)_[A-Za-z0-9]{32}|[a-f0-9]{64})$`)
|
||||||
|
|
||||||
// Config holds all agent bootstrap configuration.
|
// Config holds all agent bootstrap configuration.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
ServerURL string `env:"BACKUP_SERVER_URL,required" envDefault:"https://api.backupy.ru"`
|
ServerURL string `env:"BACKUPY_SERVER_URL,required" envDefault:"https://api.backupy.ru"`
|
||||||
AgentKey string `env:"BACKUP_AGENT_KEY,required" json:"-"`
|
AgentKey string `env:"BACKUPY_AGENT_KEY,required" json:"-"`
|
||||||
StateDir string `env:"BACKUP_STATE_DIR" envDefault:"/var/lib/backup-agent"`
|
StateDir string `env:"BACKUPY_STATE_DIR" envDefault:"/var/lib/backup-agent"`
|
||||||
LogLevel string `env:"BACKUP_LOG_LEVEL" envDefault:"info"`
|
LogLevel string `env:"BACKUPY_LOG_LEVEL" envDefault:"info"`
|
||||||
DockerSocket string `env:"BACKUP_DOCKER_SOCKET" envDefault:"/var/run/docker.sock"`
|
DockerSocket string `env:"BACKUPY_DOCKER_SOCKET" envDefault:"/var/run/docker.sock"`
|
||||||
|
|
||||||
// DevAllowInsecure relaxes the https:// requirement on ServerURL.
|
// DevAllowInsecure relaxes the https:// requirement on ServerURL.
|
||||||
// Intended for local development against a plaintext server only.
|
// Intended for local development against a plaintext server only.
|
||||||
DevAllowInsecure bool `env:"BACKUP_DEV_ALLOW_INSECURE" envDefault:"false"`
|
DevAllowInsecure bool `env:"BACKUPY_DEV_ALLOW_INSECURE" envDefault:"false"`
|
||||||
|
|
||||||
// MetricsListenAddr is the bind address for the Prometheus
|
// MetricsListenAddr is the bind address for the Prometheus
|
||||||
// `/metrics` endpoint (D-19). Default is loopback only —
|
// `/metrics` endpoint (D-19). Default is loopback only —
|
||||||
// 127.0.0.1:9090. Set to empty to disable the metrics server.
|
// 127.0.0.1:9090. Set to empty to disable the metrics server.
|
||||||
// SECURITY: never bind to 0.0.0.0 in production; the endpoint
|
// SECURITY: never bind to 0.0.0.0 in production; the endpoint
|
||||||
// reveals job IDs and run cadence usable for host fingerprinting.
|
// reveals job IDs and run cadence usable for host fingerprinting.
|
||||||
MetricsListenAddr string `env:"BACKUP_METRICS_LISTEN_ADDR" envDefault:"127.0.0.1:9090"`
|
MetricsListenAddr string `env:"BACKUPY_METRICS_LISTEN_ADDR" envDefault:"127.0.0.1:9090"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load parses environment variables into a Config and validates them.
|
// Load parses environment variables into a Config and validates them.
|
||||||
|
|
@ -62,7 +62,7 @@ func Load() (*Config, error) {
|
||||||
// Validate enforces the documented constraints on each field.
|
// Validate enforces the documented constraints on each field.
|
||||||
//
|
//
|
||||||
// - ServerURL must parse as an https:// URL (http:// only with
|
// - ServerURL must parse as an https:// URL (http:// only with
|
||||||
// BACKUP_DEV_ALLOW_INSECURE=true).
|
// BACKUPY_DEV_ALLOW_INSECURE=true).
|
||||||
// - AgentKey must match the canonical `bkpy_(live|test)_…` pattern.
|
// - AgentKey must match the canonical `bkpy_(live|test)_…` pattern.
|
||||||
// - StateDir must be writable; we test by creating and removing a temp
|
// - StateDir must be writable; we test by creating and removing a temp
|
||||||
// file so a misconfigured volume mount fails fast at startup.
|
// file so a misconfigured volume mount fails fast at startup.
|
||||||
|
|
@ -71,7 +71,7 @@ func (c *Config) Validate() error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !agentKeyPattern.MatchString(c.AgentKey) {
|
if !agentKeyPattern.MatchString(c.AgentKey) {
|
||||||
return errors.New("config: BACKUP_AGENT_KEY has invalid format; expected bkpy_(live|test)_<32 alnum>")
|
return errors.New("config: BACKUPY_AGENT_KEY has invalid format; expected 64 hex chars (or legacy bkpy_(live|test)_<32 alnum>)")
|
||||||
}
|
}
|
||||||
if err := validateStateDirWritable(c.StateDir); err != nil {
|
if err := validateStateDirWritable(c.StateDir); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|
@ -82,27 +82,27 @@ func (c *Config) Validate() error {
|
||||||
func validateServerURL(raw string, allowInsecure bool) error {
|
func validateServerURL(raw string, allowInsecure bool) error {
|
||||||
u, err := url.Parse(raw)
|
u, err := url.Parse(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("config: BACKUP_SERVER_URL is not a valid URL: %w", err)
|
return fmt.Errorf("config: BACKUPY_SERVER_URL is not a valid URL: %w", err)
|
||||||
}
|
}
|
||||||
if u.Host == "" {
|
if u.Host == "" {
|
||||||
return errors.New("config: BACKUP_SERVER_URL is missing host")
|
return errors.New("config: BACKUPY_SERVER_URL is missing host")
|
||||||
}
|
}
|
||||||
switch u.Scheme {
|
switch u.Scheme {
|
||||||
case "https":
|
case "https":
|
||||||
return nil
|
return nil
|
||||||
case "http":
|
case "http":
|
||||||
if !allowInsecure {
|
if !allowInsecure {
|
||||||
return errors.New("config: BACKUP_SERVER_URL must use https:// (set BACKUP_DEV_ALLOW_INSECURE=true for local dev)")
|
return errors.New("config: BACKUPY_SERVER_URL must use https:// (set BACKUPY_DEV_ALLOW_INSECURE=true for local dev)")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("config: BACKUP_SERVER_URL has unsupported scheme %q (expected https)", u.Scheme)
|
return fmt.Errorf("config: BACKUPY_SERVER_URL has unsupported scheme %q (expected https)", u.Scheme)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateStateDirWritable(dir string) error {
|
func validateStateDirWritable(dir string) error {
|
||||||
if dir == "" {
|
if dir == "" {
|
||||||
return errors.New("config: BACKUP_STATE_DIR must not be empty")
|
return errors.New("config: BACKUPY_STATE_DIR must not be empty")
|
||||||
}
|
}
|
||||||
// Ensure the directory exists; create it (and parents) if missing.
|
// Ensure the directory exists; create it (and parents) if missing.
|
||||||
// 0o700 — only the agent UID should ever touch state.
|
// 0o700 — only the agent UID should ever touch state.
|
||||||
|
|
|
||||||
|
|
@ -77,7 +77,7 @@ func TestValidate_AgentKey(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
} else {
|
} else {
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
require.Contains(t, err.Error(), "BACKUP_AGENT_KEY")
|
require.Contains(t, err.Error(), "BACKUPY_AGENT_KEY")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
// are eventually streamed to the server, so structured form is mandatory).
|
// are eventually streamed to the server, so structured form is mandatory).
|
||||||
// The dev profile lowers verbosity by disabling source positions.
|
// The dev profile lowers verbosity by disabling source positions.
|
||||||
//
|
//
|
||||||
// BACKUP_AGENT_KEY is never logged — see config.Config which tags it
|
// BACKUPY_AGENT_KEY is never logged — see config.Config which tags it
|
||||||
// `json:"-"` and the redactKey helper here for defence-in-depth.
|
// `json:"-"` and the redactKey helper here for defence-in-depth.
|
||||||
package logging
|
package logging
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,10 @@ type mockRunner struct {
|
||||||
outputResp map[string][]byte // key = first arg (e.g. "--version")
|
outputResp map[string][]byte // key = first arg (e.g. "--version")
|
||||||
streamResp []byte
|
streamResp []byte
|
||||||
streamErr error
|
streamErr error
|
||||||
|
// streamSideEffect, when non-nil, runs before streamResp is written
|
||||||
|
// to the supplied writer. Lets sqlite-style drivers that write to a
|
||||||
|
// real file-system path simulate the file write.
|
||||||
|
streamSideEffect func(args []string) error
|
||||||
calls []mockCall
|
calls []mockCall
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -41,6 +45,11 @@ func (m *mockRunner) RunStream(_ context.Context, _ string, args []string, env [
|
||||||
if m.streamErr != nil {
|
if m.streamErr != nil {
|
||||||
return m.streamErr
|
return m.streamErr
|
||||||
}
|
}
|
||||||
|
if m.streamSideEffect != nil {
|
||||||
|
if err := m.streamSideEffect(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(m.streamResp) > 0 {
|
if len(m.streamResp) > 0 {
|
||||||
_, _ = out.Write(m.streamResp)
|
_, _ = out.Write(m.streamResp)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|
@ -137,17 +138,24 @@ func (r *Runner) Run(ctx context.Context, req *backupv1.RunBackup) (completed *b
|
||||||
return nil, fmt.Errorf("pipeline: no driver registered for db_type=%s", driverKey)
|
return nil, fmt.Errorf("pipeline: no driver registered for db_type=%s", driverKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unwrap the DEK once. The plaintext DEK never leaves this function.
|
// Resolve the encryption stage. Jobs with encryption_enabled=false
|
||||||
|
// arrive with EncryptedDek=nil; in that case we wire the compressed
|
||||||
|
// stream straight to the uploader without ever materialising a
|
||||||
|
// plaintext DEK or instantiating an encryptor.
|
||||||
|
encryptEnabled := len(req.EncryptedDek) > 0
|
||||||
|
var encryptor *Encryptor
|
||||||
|
if encryptEnabled {
|
||||||
dek, err := r.dekResolver.Unwrap(ctx, req.EncryptedDek)
|
dek, err := r.dekResolver.Unwrap(ctx, req.EncryptedDek)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("pipeline: unwrap DEK: %w", err)
|
return nil, fmt.Errorf("pipeline: unwrap DEK: %w", err)
|
||||||
}
|
}
|
||||||
defer wipe(dek)
|
defer wipe(dek)
|
||||||
|
|
||||||
encryptor, err := NewEncryptor(dek)
|
encryptor, err = NewEncryptor(dek)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("pipeline: build encryptor: %w", err)
|
return nil, fmt.Errorf("pipeline: build encryptor: %w", err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Smoke-validate the driver before we burn upload time on a dead db.
|
// Smoke-validate the driver before we burn upload time on a dead db.
|
||||||
if err := driver.Validate(ctx, target); err != nil {
|
if err := driver.Validate(ctx, target); err != nil {
|
||||||
|
|
@ -290,9 +298,20 @@ func (r *Runner) Run(ctx context.Context, req *backupv1.RunBackup) (completed *b
|
||||||
errs <- nil
|
errs <- nil
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Stage 3 — encrypt.
|
// Stage 3 — encrypt (skipped when the job has encryption disabled;
|
||||||
|
// in that case the compressed bytes are passed through unchanged).
|
||||||
go func() {
|
go func() {
|
||||||
defer encryptedPW.Close()
|
defer encryptedPW.Close()
|
||||||
|
if encryptor == nil {
|
||||||
|
if _, err := io.Copy(encryptedPW, compressedPR); err != nil {
|
||||||
|
_ = encryptedPW.CloseWithError(err)
|
||||||
|
_ = compressedPR.CloseWithError(err)
|
||||||
|
errs <- fmt.Errorf("encrypt: passthrough copy: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
errs <- nil
|
||||||
|
return
|
||||||
|
}
|
||||||
if _, err := encryptor.Stream(compressedPR, encryptedPW); err != nil {
|
if _, err := encryptor.Stream(compressedPR, encryptedPW); err != nil {
|
||||||
_ = encryptedPW.CloseWithError(err)
|
_ = encryptedPW.CloseWithError(err)
|
||||||
_ = compressedPR.CloseWithError(err)
|
_ = compressedPR.CloseWithError(err)
|
||||||
|
|
@ -302,14 +321,30 @@ func (r *Runner) Run(ctx context.Context, req *backupv1.RunBackup) (completed *b
|
||||||
errs <- nil
|
errs <- nil
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Stage 4 — upload (blocking call on the calling goroutine). On
|
// Stage 4 — drain the encrypted pipe into a temp file, then PUT it
|
||||||
// failure we still need to wait for the three upstream goroutines
|
// with a known Content-Length. MinIO (and stricter S3 endpoints)
|
||||||
// to unwind so the function does not leak them; closing the pipe
|
// reject chunked PUTs against presigned URLs with HTTP 411.
|
||||||
// readers below makes their pending Write calls return promptly.
|
// Buffering on disk keeps memory flat while still allowing the
|
||||||
sha256hex, uploaded, uploadErr := r.uploader.Put(ctx, req.UploadCreds.PresignedPutUrl, encryptedPR, -1)
|
// dump → compress → encrypt goroutines to overlap with the drain.
|
||||||
|
stagedSize, stagedPath, stageErr := stageEncryptedBody(encryptedPR)
|
||||||
|
if stagedPath != "" {
|
||||||
|
defer func() { _ = os.Remove(stagedPath) }()
|
||||||
|
}
|
||||||
|
var sha256hex string
|
||||||
|
var uploaded int64
|
||||||
|
var uploadErr error
|
||||||
|
if stageErr != nil {
|
||||||
|
uploadErr = stageErr
|
||||||
|
} else {
|
||||||
|
stagedFile, openErr := os.Open(stagedPath)
|
||||||
|
if openErr != nil {
|
||||||
|
uploadErr = fmt.Errorf("open staged body: %w", openErr)
|
||||||
|
} else {
|
||||||
|
sha256hex, uploaded, uploadErr = r.uploader.Put(ctx, req.UploadCreds.PresignedPutUrl, stagedFile, stagedSize)
|
||||||
|
_ = stagedFile.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
if uploadErr != nil {
|
if uploadErr != nil {
|
||||||
// Closing the readers signals every upstream Write to fail
|
|
||||||
// with io.ErrClosedPipe so the producer goroutines exit.
|
|
||||||
_ = encryptedPR.CloseWithError(uploadErr)
|
_ = encryptedPR.CloseWithError(uploadErr)
|
||||||
_ = compressedPR.CloseWithError(uploadErr)
|
_ = compressedPR.CloseWithError(uploadErr)
|
||||||
_ = dumpPR.CloseWithError(uploadErr)
|
_ = dumpPR.CloseWithError(uploadErr)
|
||||||
|
|
@ -469,6 +504,29 @@ func (passthroughDEK) Unwrap(_ context.Context, in []byte) ([]byte, error) {
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stageEncryptedBody drains src into a fresh temp file and returns the
|
||||||
|
// path + total size so the caller can issue a PUT with an explicit
|
||||||
|
// Content-Length. MinIO and stricter S3 endpoints reject chunked
|
||||||
|
// transfer-encoding against presigned URLs (HTTP 411). On error, the
|
||||||
|
// caller is responsible for removing the (possibly partial) file at
|
||||||
|
// the returned path.
|
||||||
|
func stageEncryptedBody(src io.Reader) (int64, string, error) {
|
||||||
|
f, err := os.CreateTemp(os.TempDir(), "backupy-upload-*.bin")
|
||||||
|
if err != nil {
|
||||||
|
return 0, "", fmt.Errorf("stage upload: create temp: %w", err)
|
||||||
|
}
|
||||||
|
path := f.Name()
|
||||||
|
n, copyErr := io.Copy(f, src)
|
||||||
|
closeErr := f.Close()
|
||||||
|
if copyErr != nil {
|
||||||
|
return n, path, fmt.Errorf("stage upload: copy: %w", copyErr)
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return n, path, fmt.Errorf("stage upload: close: %w", closeErr)
|
||||||
|
}
|
||||||
|
return n, path, nil
|
||||||
|
}
|
||||||
|
|
||||||
// wipe zeroes a byte slice. Best-effort — the Go runtime makes no
|
// wipe zeroes a byte slice. Best-effort — the Go runtime makes no
|
||||||
// guarantee that the underlying memory pages aren't already swapped
|
// guarantee that the underlying memory pages aren't already swapped
|
||||||
// out, but this still raises the bar for casual memory inspection.
|
// out, but this still raises the bar for casual memory inspection.
|
||||||
|
|
|
||||||
|
|
@ -263,3 +263,44 @@ func TestRunner_DEKWrongLength(t *testing.T) {
|
||||||
_, err := runner.Run(context.Background(), req)
|
_, err := runner.Run(context.Background(), req)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestRunner_HappyPath_EncryptionDisabled verifies that a RunBackup
|
||||||
|
// arriving without a DEK (encryption_enabled=false on the job) skips
|
||||||
|
// the encrypt stage entirely and uploads the compressed bytes as-is.
|
||||||
|
func TestRunner_HappyPath_EncryptionDisabled(t *testing.T) {
|
||||||
|
plaintext := append([]byte(PgDumpMagic), make([]byte, 1<<10)...)
|
||||||
|
_, err := rand.Read(plaintext[len(PgDumpMagic):])
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
driver := &fakeDriver{name: "pg_dump", payload: plaintext, version: "PostgreSQL 16.2"}
|
||||||
|
job := &backupv1.BackupJobSpec{Id: "j", TargetId: "t"}
|
||||||
|
target := &backupv1.Target{Id: "t", Type: backupv1.DbType_POSTGRESQL, Connection: &backupv1.ConnectionConfig{Host: "x"}}
|
||||||
|
lookups := &simpleLookups{job: job, target: target}
|
||||||
|
|
||||||
|
var received bytes.Buffer
|
||||||
|
srv := startFakeS3(t, &received)
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
runner := NewRunner(
|
||||||
|
map[string]Driver{"postgresql": driver},
|
||||||
|
NewUploaderWithClient(srv.Client()),
|
||||||
|
WithTargetLookup(lookups),
|
||||||
|
WithJobLookup(lookups),
|
||||||
|
)
|
||||||
|
req := &backupv1.RunBackup{
|
||||||
|
JobId: "j", RunId: "r",
|
||||||
|
// No EncryptedDek — encryption disabled.
|
||||||
|
UploadCreds: &backupv1.S3UploadCreds{PresignedPutUrl: srv.URL + "/r.enc", FinalS3Key: "k"},
|
||||||
|
}
|
||||||
|
completed, err := runner.Run(context.Background(), req)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, completed.EncryptedDek, "no DEK should be reported back when encryption is disabled")
|
||||||
|
|
||||||
|
// The uploaded blob is the raw zstd stream — decompress directly.
|
||||||
|
zr, err := zstd.NewReader(&received)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer zr.Close()
|
||||||
|
round, err := io.ReadAll(zr)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, plaintext, round)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -98,17 +98,34 @@ func (s *sqliteDriver) Dump(ctx context.Context, target *backupv1.Target, out io
|
||||||
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: cannot stat database %q: %w", path, err)
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: cannot stat database %q: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
gz := gzip.NewWriter(out)
|
// `.backup` requires a regular file as the destination — the Alpine
|
||||||
defer gz.Close()
|
// sqlite3 binary refuses /dev/stdout for non-root processes. We
|
||||||
|
// stage the snapshot in a temp file and stream it to `out` after.
|
||||||
|
tmpDir := os.TempDir()
|
||||||
|
tmpFile, err := os.CreateTemp(tmpDir, "sqlite-backup-*.db")
|
||||||
|
if err != nil {
|
||||||
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: create temp file: %w", err)
|
||||||
|
}
|
||||||
|
tmpPath := tmpFile.Name()
|
||||||
|
_ = tmpFile.Close()
|
||||||
|
defer func() { _ = os.Remove(tmpPath) }()
|
||||||
|
|
||||||
// sqlite3 expects a single positional argument (the database path)
|
args := []string{path, fmt.Sprintf(".backup '%s'", tmpPath)}
|
||||||
// followed by a dot-command. `.backup` writes a consistent snapshot
|
if err := s.runner.RunStream(ctx, s.binary, args, nil, io.Discard); err != nil {
|
||||||
// to the supplied filename; we pass /dev/stdout so the bytes flow
|
|
||||||
// through stdout into our gzip writer.
|
|
||||||
args := []string{path, ".backup '/dev/stdout'"}
|
|
||||||
if err := s.runner.RunStream(ctx, s.binary, args, nil, gz); err != nil {
|
|
||||||
return DumpInfo{}, fmt.Errorf("pipeline: sqlite3 .backup exec: %w", err)
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite3 .backup exec: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
src, err := os.Open(tmpPath)
|
||||||
|
if err != nil {
|
||||||
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: open snapshot: %w", err)
|
||||||
|
}
|
||||||
|
defer src.Close()
|
||||||
|
|
||||||
|
gz := gzip.NewWriter(out)
|
||||||
|
if _, err := io.Copy(gz, src); err != nil {
|
||||||
|
_ = gz.Close()
|
||||||
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: gzip copy: %w", err)
|
||||||
|
}
|
||||||
if err := gz.Close(); err != nil {
|
if err := gz.Close(); err != nil {
|
||||||
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: close gzip: %w", err)
|
return DumpInfo{}, fmt.Errorf("pipeline: sqlite: close gzip: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
|
@ -91,7 +92,23 @@ func TestSqlite_Dump_WrapsOutputInGzip(t *testing.T) {
|
||||||
payload := bytes.Repeat([]byte{0x53, 0x51, 0x4c}, 16) // pretend SQLite header bytes
|
payload := bytes.Repeat([]byte{0x53, 0x51, 0x4c}, 16) // pretend SQLite header bytes
|
||||||
mock := &mockRunner{
|
mock := &mockRunner{
|
||||||
outputResp: map[string][]byte{"--version": []byte("3.45.1 2024-01-30\n")},
|
outputResp: map[string][]byte{"--version": []byte("3.45.1 2024-01-30\n")},
|
||||||
streamResp: payload,
|
// The new Dump implementation stages the snapshot in a temp
|
||||||
|
// file, then re-reads it into the gzip pipe. Simulate the
|
||||||
|
// real sqlite3 process by parsing the destination path out
|
||||||
|
// of the .backup dot-command and writing the canned payload
|
||||||
|
// to that path.
|
||||||
|
streamSideEffect: func(args []string) error {
|
||||||
|
if len(args) < 2 {
|
||||||
|
return errors.New("expected at least 2 args")
|
||||||
|
}
|
||||||
|
dot := args[1]
|
||||||
|
const prefix = ".backup '"
|
||||||
|
if !strings.HasPrefix(dot, prefix) || !strings.HasSuffix(dot, "'") {
|
||||||
|
return errors.New("malformed .backup dot-command")
|
||||||
|
}
|
||||||
|
path := dot[len(prefix) : len(dot)-1]
|
||||||
|
return os.WriteFile(path, payload, 0o600)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
d := &sqliteDriver{binary: "sqlite3", runner: mock, statFn: os.Stat}
|
d := &sqliteDriver{binary: "sqlite3", runner: mock, statFn: os.Stat}
|
||||||
|
|
||||||
|
|
@ -111,11 +128,12 @@ func TestSqlite_Dump_WrapsOutputInGzip(t *testing.T) {
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, payload, got)
|
require.Equal(t, payload, got)
|
||||||
|
|
||||||
// Confirm `.backup '/dev/stdout'` was invoked with the right path.
|
// Confirm the dump invoked `.backup '<tmpfile>'`.
|
||||||
require.NotEmpty(t, mock.calls)
|
require.NotEmpty(t, mock.calls)
|
||||||
streamCall := mock.calls[0]
|
streamCall := mock.calls[0]
|
||||||
require.Equal(t, tmp, streamCall.Args[0])
|
require.Equal(t, tmp, streamCall.Args[0])
|
||||||
require.Equal(t, ".backup '/dev/stdout'", streamCall.Args[1])
|
require.True(t, strings.HasPrefix(streamCall.Args[1], ".backup '"))
|
||||||
|
require.True(t, strings.HasSuffix(streamCall.Args[1], "'"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSqlite_Dump_MissingPath(t *testing.T) {
|
func TestSqlite_Dump_MissingPath(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
// State-at-rest encryption helpers.
|
// State-at-rest encryption helpers.
|
||||||
//
|
//
|
||||||
// All bucket *values* are wrapped with AES-256-GCM using a key derived from
|
// All bucket *values* are wrapped with AES-256-GCM using a key derived from
|
||||||
// BACKUP_AGENT_KEY via HKDF-SHA256 (per docs/03-agent-spec.md →
|
// BACKUPY_AGENT_KEY via HKDF-SHA256 (per docs/03-agent-spec.md →
|
||||||
// "Шифрование state опционально (key derived из BACKUP_AGENT_KEY)").
|
// "Шифрование state опционально (key derived из BACKUPY_AGENT_KEY)").
|
||||||
//
|
//
|
||||||
// Wire format on disk:
|
// Wire format on disk:
|
||||||
//
|
//
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
// Package state owns the agent's persistent on-disk state — a BoltDB file
|
// Package state owns the agent's persistent on-disk state — a BoltDB file
|
||||||
// at $BACKUP_STATE_DIR/state.db.
|
// at $BACKUPY_STATE_DIR/state.db.
|
||||||
//
|
//
|
||||||
// Buckets:
|
// Buckets:
|
||||||
//
|
//
|
||||||
|
|
@ -8,7 +8,7 @@
|
||||||
// "registry" — session metadata: last session_id, server_time, heartbeat.
|
// "registry" — session metadata: last session_id, server_time, heartbeat.
|
||||||
// "logs_buffer" — rate-limited LogEvent buffer when server is unreachable.
|
// "logs_buffer" — rate-limited LogEvent buffer when server is unreachable.
|
||||||
//
|
//
|
||||||
// All bucket values are encrypted with AES-256-GCM keyed by HKDF(BACKUP_AGENT_KEY).
|
// All bucket values are encrypted with AES-256-GCM keyed by HKDF(BACKUPY_AGENT_KEY).
|
||||||
// See crypto.go for the wire format.
|
// See crypto.go for the wire format.
|
||||||
//
|
//
|
||||||
// Concurrency: bbolt serialises write transactions itself, so the Store is
|
// Concurrency: bbolt serialises write transactions itself, so the Store is
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ type Config struct {
|
||||||
Capabilities []string
|
Capabilities []string
|
||||||
// AllowInsecure permits ws:// / http:// dial schemes when ServerURL
|
// AllowInsecure permits ws:// / http:// dial schemes when ServerURL
|
||||||
// uses one. Production must leave this false — it matches the
|
// uses one. Production must leave this false — it matches the
|
||||||
// agent's BACKUP_DEV_ALLOW_INSECURE bootstrap flag.
|
// agent's BACKUPY_DEV_ALLOW_INSECURE bootstrap flag.
|
||||||
AllowInsecure bool
|
AllowInsecure bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
32
docker-compose.example.yml
Normal file
32
docker-compose.example.yml
Normal file
|
|
@ -0,0 +1,32 @@
|
||||||
|
# Backupy agent — paste-ready compose snippet.
|
||||||
|
#
|
||||||
|
# 1. Sign up at https://backupy.tronosfera.ru
|
||||||
|
# 2. Dashboard -> Agents -> Add agent. Copy the one-time key.
|
||||||
|
# 3. Save the key in `.env` next to this file:
|
||||||
|
# BACKUPY_AGENT_KEY=bk_agent_xxxxxxxxxxxxxxxxxxxx
|
||||||
|
# 4. docker compose up -d backupy-agent
|
||||||
|
#
|
||||||
|
# The agent connects out to wss://backupy.tronosfera.ru — no inbound
|
||||||
|
# ports needed.
|
||||||
|
|
||||||
|
services:
|
||||||
|
backupy-agent:
|
||||||
|
image: ghcr.io/tronosfera/backupy-agent:0.1.0
|
||||||
|
container_name: backupy-agent
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
BACKUPY_SERVER_URL: wss://backupy.tronosfera.ru/agents/connect
|
||||||
|
BACKUPY_AGENT_KEY: ${BACKUPY_AGENT_KEY}
|
||||||
|
# Set to "true" to skip Docker socket discovery (jobs still work,
|
||||||
|
# but no auto-detection of databases running on this host).
|
||||||
|
# BACKUPY_DISABLE_DISCOVERY: "false"
|
||||||
|
volumes:
|
||||||
|
# Read-only socket for Docker discovery. Drop this volume mount if
|
||||||
|
# you don't want the agent to enumerate sibling containers.
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
# Persistent state (BoltDB queue + last-seen offsets).
|
||||||
|
- backupy_agent:/var/lib/backupy
|
||||||
|
# The agent runs as uid 1000 by default and is otherwise unprivileged.
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
backupy_agent:
|
||||||
|
|
@ -11,8 +11,8 @@ services:
|
||||||
backup-agent:
|
backup-agent:
|
||||||
image: backupservice/agent:latest
|
image: backupservice/agent:latest
|
||||||
environment:
|
environment:
|
||||||
BACKUP_SERVER_URL: https://backupy.ru
|
BACKUPY_SERVER_URL: https://backupy.ru
|
||||||
BACKUP_AGENT_KEY: ${BACKUP_KEY}
|
BACKUPY_AGENT_KEY: ${BACKUP_KEY}
|
||||||
volumes:
|
volumes:
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
- backup-agent-state:/var/lib/backup-agent
|
- backup-agent-state:/var/lib/backup-agent
|
||||||
|
|
@ -26,10 +26,10 @@ volumes:
|
||||||
|
|
||||||
| Имя | Назначение | Required |
|
| Имя | Назначение | Required |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `BACKUP_SERVER_URL` | Адрес control plane | да |
|
| `BACKUPY_SERVER_URL` | Адрес control plane | да |
|
||||||
| `BACKUP_AGENT_KEY` | Ключ агента (секрет) | да |
|
| `BACKUPY_AGENT_KEY` | Ключ агента (секрет) | да |
|
||||||
| `BACKUP_LOG_LEVEL` | trace/debug/info/warn/error, default info | нет |
|
| `BACKUPY_LOG_LEVEL` | trace/debug/info/warn/error, default info | нет |
|
||||||
| `BACKUP_STATE_DIR` | Путь к state, default `/var/lib/backup-agent` | нет |
|
| `BACKUPY_STATE_DIR` | Путь к state, default `/var/lib/backup-agent` | нет |
|
||||||
|
|
||||||
Всё остальное (targets, schedules, S3 creds, retention, hooks) — приходит с сервера через `ConfigUpdate`.
|
Всё остальное (targets, schedules, S3 creds, retention, hooks) — приходит с сервера через `ConfigUpdate`.
|
||||||
|
|
||||||
|
|
@ -46,7 +46,7 @@ volumes:
|
||||||
### Persistent state в volume
|
### Persistent state в volume
|
||||||
- SQLite или BoltDB в `/var/lib/backup-agent/state.db`.
|
- SQLite или BoltDB в `/var/lib/backup-agent/state.db`.
|
||||||
- Хранит: текущий config, очередь jobs, локальные логи, последний known config_version.
|
- Хранит: текущий config, очередь jobs, локальные логи, последний known config_version.
|
||||||
- Шифрование state опционально (key derived из BACKUP_AGENT_KEY).
|
- Шифрование state опционально (key derived из BACKUPY_AGENT_KEY).
|
||||||
|
|
||||||
### WSS-канал
|
### WSS-канал
|
||||||
- Один long-lived connection на agent_id.
|
- Один long-lived connection на agent_id.
|
||||||
|
|
@ -114,7 +114,7 @@ volumes:
|
||||||
- TLS 1.3 ко всем endpoint'ам.
|
- TLS 1.3 ко всем endpoint'ам.
|
||||||
- Pinning публичного ключа сервера (зашит в бинарь).
|
- Pinning публичного ключа сервера (зашит в бинарь).
|
||||||
- Docker socket монтируется read-only.
|
- Docker socket монтируется read-only.
|
||||||
- `BACKUP_AGENT_KEY` никогда не пишется в логи.
|
- `BACKUPY_AGENT_KEY` никогда не пишется в логи.
|
||||||
- Локальный state шифруется (опционально включается).
|
- Локальный state шифруется (опционально включается).
|
||||||
- Healthcheck endpoint (если будет) — только на localhost.
|
- Healthcheck endpoint (если будет) — только на localhost.
|
||||||
- Capabilities контейнера: drop ALL.
|
- Capabilities контейнера: drop ALL.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue