teqo-backup/scripts/deploy-homeserver.sh

#!/usr/bin/env bash
#
# Deploy teqo-1313 (production) or teqo-staging to the homeserver (OPS53,
# parameterized by OPS103).
#
# OPS71: runs ON the homeserver via the GitHub self-hosted runner (deploy jobs
# of .github/workflows/deploy.yml):
#   bash scripts/deploy-homeserver.sh <commit-sha>
# (The Forgejo-era invocation `ssh homeserver "bash -s -- <sha>" < script`
# is gone — no SSH hop, no workstation involvement.)
#
# OPS103: one script, both environments — `TEQO_ENV=production` (default,
# byte-for-byte the pre-OPS103 behavior) and `TEQO_ENV=staging`. Each
# environment owns its container/service, env file, image repo, build-proxy
# port, smoke base and lock; the compose file, workspace and registry stay
# shared. An unknown TEQO_ENV fails closed before anything is touched.
#
# OPS102: a manual dispatch runs to the end with its SHA even if main advances
# during verify (the old stale-run guard belonged to the automatic era); the
# only early green is the idempotency guard, which proves the SHA is already
# running.
#
# Flow: flock serialization -> workspace fetch at <sha> -> docker login
# (local registry) -> build of the MIGRATOR stage (it never runs `next
# build`, so it builds even against the old schema) -> push of the migrator
# (registry-qualified tag — INF13: the ONLY ref the compose references) ->
# compose image-tag swap
# (with backup) -> migrate via the maintenance service
# (BEFORE the runner build — static generation reads the NEW schema, OPS66)
# -> build of the runner stage (BuildKit secrets, compose network) ->
# push -> compose up -> healthcheck wait -> smoke. Any failure after the
# swap rolls back to the previous compose + image; failures before it leave
# the running site untouched and the job red.
#
# Environment defaults assume the homeserver layout: stack under
# $HOME/stack, repo cloned from github.com/fsolla/teqo (public — no
# credentials; the pre-OPS71 clone pointed at the local Forgejo and is
# re-pointed idempotently below). Secrets are sourced from the chmod-600 env
# files and never echoed (no `set -x`, passwords only via --password-stdin /
# build secrets).

set -euo pipefail

SHA="${1:?usage: deploy-homeserver.sh <commit-sha>}"
TEQO_ENV="${TEQO_ENV:-production}"
TEQO_REPO_URL="${TEQO_REPO_URL:-https://github.com/fsolla/teqo.git}"
STACK_DIR="${STACK_DIR:-$HOME/stack}"
WORKSPACE_DIR="${WORKSPACE_DIR:-$HOME/teqo-deploy}"
TEQO_REGISTRY="${TEQO_REGISTRY:-localhost:5000}"
# ONE lock for both environments: the compose file, the workspace and the
# local registry are shared, so staging and production must never build or
# swap concurrently (the workflow's job-level concurrency lives only on
# deploy-staging since OPS107 — the production approval never holds it, so
# this lock IS the cross-run serialization; it also covers manual invocations
# on the host).
DEPLOY_LOCK="${DEPLOY_LOCK:-/tmp/teqo-deploy.lock}"

say() { printf '[deploy] %s\n' "$*"; }

fatal() {
  trap - ERR
  say "FAILED: $*" >&2
  exit 1
}

probe_media_endpoint() {
  local image="$1"
  local endpoint="$2"
  local host="${3:-}"
  local -a args=(--rm --network stack_default)
  if [ -n "$host" ]; then
    args+=(--add-host "host.docker.internal:$host")
  fi
  docker run "${args[@]}" \
    --env "S3_PROBE_ENDPOINT=$endpoint" \
    --entrypoint node \
    "$image" \
    -e 'fetch(process.env.S3_PROBE_ENDPOINT, { signal: AbortSignal.timeout(5000) }).then(() => process.exit(0)).catch(() => process.exit(1))' \
    >/dev/null 2>&1
}

media_gateways() {
  docker network inspect -f '{{range .IPAM.Config}}{{println .Gateway}}{{end}}' stack_default 2>/dev/null
}

ensure_media_endpoint() {
  local image="$TEQO_REGISTRY/$TEQO_IMAGE_REPO-migrator:$SHA"
  local override="$STACK_DIR/docker-compose.media-$TEQO_ENV.yml"

  if [ -z "${S3_BUCKET:-}" ] || [ -z "${S3_ENDPOINT:-}" ] || [ -z "${S3_ACCESS_KEY_ID:-}" ] || [ -z "${S3_SECRET_ACCESS_KEY:-}" ]; then
    rm -f "$override"
    unset COMPOSE_FILE
    return 0
  fi

  if ! node "$WORKSPACE_DIR/scripts/lib/mediaEndpoint.mjs" is-host-gateway --endpoint "$S3_ENDPOINT"; then
    if probe_media_endpoint "$image" "$S3_ENDPOINT" ""; then
      rm -f "$override"
      unset COMPOSE_FILE
      return 0
    fi
    fatal "configured S3 endpoint is unreachable from the Docker network"
  fi

  if probe_media_endpoint "$image" "$S3_ENDPOINT" "host-gateway"; then
    rm -f "$override"
    unset COMPOSE_FILE
    return 0
  fi

  local gateway
  while IFS= read -r gateway; do
    if [[ ! "$gateway" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
      continue
    fi
    if probe_media_endpoint "$image" "$S3_ENDPOINT" "$gateway"; then
      if ! node "$WORKSPACE_DIR/scripts/lib/mediaEndpoint.mjs" \
        --container "$TEQO_CONTAINER" \
        --migrate-service "$TEQO_MIGRATE_SERVICE" \
        --endpoint "$S3_ENDPOINT" \
        --gateway "$gateway" >"$override"; then
        rm -f "$override"
        fatal "failed to render media endpoint override"
      fi
      if ! docker compose -f "$STACK_DIR/docker-compose.yml" -f "$override" config --quiet; then
        rm -f "$override"
        fatal "media endpoint override is invalid"
      fi
      export COMPOSE_FILE="$STACK_DIR/docker-compose.yml:$override"
      say "repairing S3 route for $TEQO_CONTAINER with the reachable stack gateway"
      return 0
    fi
  done < <(media_gateways)

  fatal "no reachable IPv4 gateway for the configured S3 endpoint"
}

smoke_media_endpoint() {
  if [ -z "${S3_BUCKET:-}" ] || [ -z "${S3_ENDPOINT:-}" ] || [ -z "${S3_ACCESS_KEY_ID:-}" ] || [ -z "${S3_SECRET_ACCESS_KEY:-}" ]; then
    return 0
  fi
  docker exec "$TEQO_CONTAINER" node -e \
    'fetch(process.env.S3_ENDPOINT, { signal: AbortSignal.timeout(5000) }).then(() => process.exit(0)).catch(() => process.exit(1))' \
    >/dev/null 2>&1 || rollback "S3 endpoint missing or unreachable from $TEQO_CONTAINER"
}

# --- environment map (OPS103) -------------------------------------------
# Production values are the pre-OPS103 literals: the canonical invocation
# `bash scripts/deploy-homeserver.sh <sha>` (no TEQO_ENV) behaves exactly as
# before. The workflow passes TEQO_ENV per deploy job. The DB name is NOT
# here — it lives inside each environment's env file. `readonly` keeps the
# env files (sourced below) from silently retargeting the deploy.

case "$TEQO_ENV" in
  production)
    TEQO_CONTAINER=teqo-1313
    TEQO_MIGRATE_SERVICE=teqo-1313-migrate
    TEQO_ENV_FILE="$STACK_DIR/teqo-1313.env"
    TEQO_IMAGE_REPO=teqo-1313
    TEQO_BUILD_PROXY=teqo-1313-build-proxy
    TEQO_BUILD_PROXY_PORT=5433
    TEQO_SMOKE_BASE=http://localhost:1313
    ;;
  staging)
    TEQO_CONTAINER=teqo-staging
    TEQO_MIGRATE_SERVICE=teqo-staging-migrate
    TEQO_ENV_FILE="$STACK_DIR/teqo-staging.env"
    TEQO_IMAGE_REPO=teqo-staging
    TEQO_BUILD_PROXY=teqo-staging-build-proxy
    TEQO_BUILD_PROXY_PORT=5434
    TEQO_SMOKE_BASE=http://localhost:1314
    ;;
  *)
    fatal "unknown TEQO_ENV '$TEQO_ENV' (expected production or staging)"
    ;;
esac

readonly TEQO_CONTAINER TEQO_MIGRATE_SERVICE TEQO_ENV_FILE TEQO_IMAGE_REPO
readonly TEQO_BUILD_PROXY TEQO_BUILD_PROXY_PORT TEQO_SMOKE_BASE DEPLOY_LOCK

# --- serialization (flock) ----------------------------------------------

exec 9>"$DEPLOY_LOCK"
flock -w 3600 9 || fatal "another deploy holds $DEPLOY_LOCK"

# --- idempotency (OPS65) ------------------------------------------------
# A duplicate workflow_dispatch can re-deliver a SHA the cluster already runs:
# rebuilding it is a ~15 min no-op. The truth
# is the RUNNING container's revision label — the compose file can lie after
# a failed rollback (it is swapped before the rollout and restored best-
# effort). A container without the label (or down) counts as "not deployed":
# the deploy proceeds (safe direction — it rebuilds).

running_rev="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$TEQO_CONTAINER" 2>/dev/null || true)"
if [ -n "$running_rev" ] && [ "$running_rev" = "$SHA" ]; then
  say "already deployed: $TEQO_CONTAINER runs $SHA — nothing to do"
  exit 0
fi

# --- secrets (sourced locally, never echoed) ----------------------------

set -a
. "$TEQO_ENV_FILE"
. "$STACK_DIR/.env"
set +a

: "${DATABASE_URL:?$TEQO_ENV_FILE is missing DATABASE_URL}"
: "${PAYLOAD_SECRET:?$TEQO_ENV_FILE is missing PAYLOAD_SECRET}"
: "${NEXT_PUBLIC_SITE_URL:?$TEQO_ENV_FILE is missing NEXT_PUBLIC_SITE_URL}"
: "${REVALIDATE_SECRET:?$TEQO_ENV_FILE is missing REVALIDATE_SECRET}"
: "${REGISTRY_USER:?stack/.env is missing REGISTRY_USER}"
: "${REGISTRY_PASSWORD:?stack/.env is missing REGISTRY_PASSWORD}"

# --- workspace at <sha> -------------------------------------------------

if [ ! -d "$WORKSPACE_DIR/.git" ]; then
  say "cloning $TEQO_REPO_URL into $WORKSPACE_DIR"
  git clone "$TEQO_REPO_URL" "$WORKSPACE_DIR"
else
  # OPS71: the pre-cutover clone pointed at the local Forgejo — re-point
  # idempotently so the fetch targets the repo that actually received the
  # merge.
  current_url="$(git -C "$WORKSPACE_DIR" remote get-url origin 2>/dev/null || true)"
  if [ -n "$current_url" ] && [ "$current_url" != "$TEQO_REPO_URL" ]; then
    say "updating workspace origin: $current_url → $TEQO_REPO_URL"
    git -C "$WORKSPACE_DIR" remote set-url origin "$TEQO_REPO_URL"
  fi
  git -C "$WORKSPACE_DIR" fetch origin main
fi
git -C "$WORKSPACE_DIR" checkout --detach "$SHA" || fatal "checkout of $SHA failed"
say "workspace at $SHA"

# --- registry -----------------------------------------------------------

say "logging into local registry $TEQO_REGISTRY"
echo "$REGISTRY_PASSWORD" | docker login "$TEQO_REGISTRY" --username "$REGISTRY_USER" --password-stdin >/dev/null

# --- build --------------------------------------------------------------

# BuildKit rejects a custom bridge network on `--network`, so the build runs
# with `--network host`; the env DB (only reachable inside stack_default) is
# proxied to the host loopback by a one-off socat container on the compose
# network, and DATABASE_URL is rewritten to that endpoint. The proxy is
# idempotent and survives reboots (restart: unless-stopped). Each
# environment publishes its own loopback port (TEQO_BUILD_PROXY_PORT) so the
# two proxies never collide.
ensure_db_proxy() {
  if ! docker inspect "$TEQO_BUILD_PROXY" >/dev/null 2>&1; then
    docker run -d --name "$TEQO_BUILD_PROXY" \
      --network stack_default \
      --restart unless-stopped \
      -p "127.0.0.1:$TEQO_BUILD_PROXY_PORT:$TEQO_BUILD_PROXY_PORT" \
      alpine/socat "TCP-LISTEN:$TEQO_BUILD_PROXY_PORT,fork TCP:postgres:5432" >/dev/null
  fi
}

build_image() {
  local target="$1" image="$2"
  say "building $target ($image)"
  ensure_db_proxy
  # The build's DATABASE_URL points at the loopback proxy; the outer env
  # keeps the original value and nothing is echoed.
  local build_db_url="${DATABASE_URL/@postgres:5432/@127.0.0.1:$TEQO_BUILD_PROXY_PORT}"
  DATABASE_URL="$build_db_url" DOCKER_BUILDKIT=1 docker build \
    --network host \
    --build-arg "NEXT_PUBLIC_SITE_URL=$NEXT_PUBLIC_SITE_URL" \
    --secret "id=database_url,env=DATABASE_URL" \
    --secret "id=payload_secret,env=PAYLOAD_SECRET" \
    --target "$target" \
    -t "$image" .
}

cd "$WORKSPACE_DIR"

# The migrator stage never runs `next build`, so it builds fine against the
# OLD schema — unlike the runner, whose static generation reads Payload data
# and therefore needs the migrations applied first (OPS66).
build_image migrator "$TEQO_REGISTRY/$TEQO_IMAGE_REPO-migrator:$SHA"

# INF13: the compose references REGISTRY-QUALIFIED tags
# (`$TEQO_REGISTRY/$TEQO_IMAGE_REPO(-migrator):<sha>`) with pull_policy: never
# — the exact name every build produces, so the ref the compose needs can
# never go missing (the old bare `teqo-1313:<sha>` alias was a second
# local-only ref that nothing recreated once lost — homeserver incident
# 24/08, `No such image` on recreate).
docker push "$TEQO_REGISTRY/$TEQO_IMAGE_REPO-migrator:$SHA"

# --- compose swap (backup first; failures from here roll back) ----------

compose="$STACK_DIR/docker-compose.yml"
ensure_media_endpoint
backup="$compose.pre-$SHA"
cp "$compose" "$backup"
# Image lines: the env-specific repo names cannot touch the other
# environment's services (the migrator pattern runs first; `teqo-1313:`
# never matches `teqo-1313-migrator:`).
sed -i -E \
  -e "s|image: ($TEQO_REGISTRY/)?$TEQO_IMAGE_REPO-migrator:[0-9a-f]+|image: $TEQO_REGISTRY/$TEQO_IMAGE_REPO-migrator:$SHA|g" \
  -e "s|image: ($TEQO_REGISTRY/)?$TEQO_IMAGE_REPO:[0-9a-f]+|image: $TEQO_REGISTRY/$TEQO_IMAGE_REPO:$SHA|g" \
  "$compose"
# Revision label: it lives INSIDE the service block, so a bare global
# replace would stamp THIS env's SHA onto the other environment's service —
# and the "already deployed" guard would read a lying revision. The sed
# range starts at the service key and ends at the next two-space key (the
# next service); keys inside a service are indented four spaces. The range
# is then re-read to fail closed when the service has no revision label
# (a silent no-op would disable the idempotency guard for this env).
label_range="/^  $TEQO_CONTAINER:/,/^  [A-Za-z0-9_.-]+:/"
sed -i -E -e "$label_range s|org.opencontainers.image.revision: [0-9a-f]+|org.opencontainers.image.revision: $SHA|" "$compose"
sed -n -e "$label_range p" "$compose" | grep -q "org.opencontainers.image.revision: $SHA" \
  || fatal "compose swap failed (revision label of $TEQO_CONTAINER)"
grep -q "image: $TEQO_REGISTRY/$TEQO_IMAGE_REPO:$SHA" "$compose" || fatal "compose swap failed (runner image)"
grep -q "image: $TEQO_REGISTRY/$TEQO_IMAGE_REPO-migrator:$SHA" "$compose" || fatal "compose swap failed (migrator image)"

rollback() {
  trap - ERR
  say "FAILED: $* — restoring previous compose and image"
  cp "$backup" "$compose" 2>/dev/null || true
  ( cd "$STACK_DIR" && docker compose up -d "$TEQO_CONTAINER" ) 2>/dev/null || true
  exit 1
}
trap 'rollback "unexpected failure"' ERR

# --- migrate (BEFORE the runner build — OPS66) --------------------------
# The runner build statically generates pages that read Payload data; a
# migration that creates a table a static route reads would otherwise
# deadlock the deploy (build fails -> migrate never runs -> build fails...).
# The migrator image above is already swapped into the compose, so this
# maintenance service runs the migrations of the NEW sha against the env DB.

cd "$STACK_DIR"
say "applying pending migrations (maintenance service $TEQO_MIGRATE_SERVICE)"
# `< /dev/null`: `compose run` attaches stdin by default; the container would
# consume the rest of the script piped into `bash -s` (bash then hits EOF and
# exits right after the migrate step, skipping the rollout).
docker compose --profile maintenance run --rm "$TEQO_MIGRATE_SERVICE" </dev/null || rollback "migrations failed"

# --- runner build (against the migrated schema) --------------------------

cd "$WORKSPACE_DIR"
say "building runner"
build_image runner "$TEQO_REGISTRY/$TEQO_IMAGE_REPO:$SHA"
docker push "$TEQO_REGISTRY/$TEQO_IMAGE_REPO:$SHA"

# --- rollout ------------------------------------------------------------

cd "$STACK_DIR"
say "rolling out $TEQO_CONTAINER"
docker compose up -d "$TEQO_CONTAINER" || rollback "compose up failed"

say "waiting for the healthcheck to go healthy"
health="starting"
for _ in $(seq 1 30); do
  health="$(docker inspect -f '{{.State.Health.Status}}' "$TEQO_CONTAINER" 2>/dev/null || true)"
  [ "$health" = "healthy" ] && break
  [ "$health" = "unhealthy" ] && break
  sleep 10
done
[ "$health" = "healthy" ] || rollback "container not healthy after 300s (status: $health)"

smoke_media_endpoint

# --- feature env audit (non-fatal) --------------------------------------
# A missing feature key never fails the boot: the capability degrades
# silently (AI theme search/titles/transcription, forgot-password e-mail,
# push). The audit reads the RUNNING container env — the compose env_file is
# the source of truth, not what the script sourced — and annotates what is
# missing so the operator sees the capability loss in this very deploy. It
# never fails the deploy: which feature keys are set is the human's call.
# The list mirrors the app predicates (`isCampaignEmailConfigured`,
# `resolveVapidConfig` and the DeepSeek/Deep Infra guards).

FEATURE_ENV_KEYS=(
  DEEPSEEK_API_KEY
  DEEPINFRA_API_KEY
  RESEND_API_KEY
  CAMPAIGN_EMAIL_FROM
  VAPID_PUBLIC_KEY
  VAPID_PRIVATE_KEY
)

warn_missing_feature_envs() {
  local container_env missing=() name
  container_env="$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$TEQO_CONTAINER" 2>/dev/null || true)"
  for name in "${FEATURE_ENV_KEYS[@]}"; do
    printf '%s\n' "$container_env" | grep -q "^${name}=." || missing+=("$name")
  done
  if [ "${#missing[@]}" -gt 0 ]; then
    say "WARNING: $TEQO_CONTAINER is missing feature envs: ${missing[*]}"
    printf '::warning::%s is missing feature envs: %s (those capabilities degrade silently)\n' \
      "$TEQO_CONTAINER" "${missing[*]}"
  fi
}

warn_missing_feature_envs

# --- AI egress probe (non-fatal) ----------------------------------------
# The AI keys can be present and the capability still dead: the homeserver's
# route to api.deepseek.com (CloudFront) is intermittently unreachable (incident
# 2026-09-25), and the app's egress is a tailnet proxy that can be down. Probe
# the ACTUAL container path (it honors NODE_USE_ENV_PROXY/HTTPS_PROXY) and warn
# (never fail: the app degrades to its fallbacks).

warn_ai_egress() {
  docker exec "$TEQO_CONTAINER" node -e "
    const controller = new AbortController()
    const timer = setTimeout(() => controller.abort(), 8000)
    fetch('https://api.deepseek.com/', { signal: controller.signal })
      .then((response) => { clearTimeout(timer); process.exit(0) })
      .catch(() => process.exit(1))
  " >/dev/null 2>&1 && return 0
  say "WARNING: $TEQO_CONTAINER cannot reach api.deepseek.com — AI features degrade to their fallbacks"
  printf '::warning::%s cannot reach api.deepseek.com (AI capabilities degrade silently)\n' "$TEQO_CONTAINER"
}

warn_ai_egress

# --- smoke --------------------------------------------------------------

base="$TEQO_SMOKE_BASE"
smoke_fail() { rollback "smoke: $*"; }

curl -fsS -o /dev/null "$base/" || smoke_fail "GET /"
curl -fsS -o /dev/null "$base/campanha/login" || smoke_fail "GET /campanha/login"
curl -fsS -o /dev/null "$base/admin" || smoke_fail "GET /admin"
code="$(curl -s -o /dev/null -w '%{http_code}' "$base/campanha" || true)"
[ "$code" = "307" ] || smoke_fail "GET /campanha expected 307, got $code"
curl -fsS -o /dev/null -X POST "$base/campanha/webauthn/login-options" \
  -H 'Content-Type: application/json' -d '{}' || smoke_fail "POST webauthn login-options"
body="$(curl -fsS -X POST "$base/api/revalidate" -H "x-revalidate-secret: $REVALIDATE_SECRET" || true)"
echo "$body" | grep -q '"revalidated":true' || smoke_fail "POST /api/revalidate did not confirm"

# --- post-deploy cleanup (INF3/F2) --------------------------------------
# O deploy compila no homeserver e acumulava build cache + tags locais
# antigas no disco raiz (incidente 19/08: 45G de cache + ~20 tags teqo
# antigas). Best-effort e APÓS o smoke: falha de limpeza nunca falha um
# deploy verde. O registry localhost:5000 preserva as imagens (rollback
# intacto — runbook teqo-1313-deploy.md); removemos só as tags LOCAIS.
# INF13: a preservação é pelo IMAGE ID do container em uso (`{{.Image}}`),
# não por comparação tag×label do compose — imune a drift de label/recriação
# manual; TODAS as tags da imagem rodando sobrevivem, as demais são removidas.
# Fail-closed: container sem image ID não remove nada além do cache.

say "post-deploy cleanup: build cache + tags locais antigas"
docker builder prune -f >/dev/null 2>&1 || true

in_use_id="$(docker inspect -f '{{.Image}}' "$TEQO_CONTAINER" 2>/dev/null || true)"
if [ -n "$in_use_id" ]; then
  for img in $(docker images --format '{{.Repository}}:{{.Tag}}' | grep -E "^($TEQO_REGISTRY/)?$TEQO_IMAGE_REPO(-migrator)?:" || true); do
    img_id="$(docker image inspect -f '{{.Id}}' "$img" 2>/dev/null || true)"
    [ -n "$img_id" ] && [ "$img_id" = "$in_use_id" ] && continue
    docker rmi "$img" >/dev/null 2>&1 || true
  done
else
  say "cleanup: container sem image ID — mantendo imagens locais"
fi

say "deploy of $SHA complete: $(docker inspect -f '{{.Image}}' "$TEQO_CONTAINER")"