#!/usr/bin/env bash
# ==============================================================================
# Plexicus On-Prem Update
#
#   curl -fsSL https://docs.plexicus.ai/update.sh -o update.sh
#   sudo -E PLEXICUS_ENV_FILE=/root/plexicus-eval.env bash update.sh
#
# Safely upgrades an EXISTING Plexicus installation to the latest released
# chart (or to an explicit --version). Covers all supported install flavors:
#
#   * evaluation   — installed by install.sh (single-node k3s VM); the upgrade
#                    re-runs the installer, which is idempotent by design.
#   * production   — installed per the Production Installation guide (own
#                    cluster, own my-values.yaml); the upgrade is a pinned
#                    `helm upgrade` with that values file.
#   * air-gapped   — cluster only reaches an internal registry; you mirror the
#                    images/chart for the new release first, then pass
#                    --version (and --registry) explicitly.
#
# ArgoCD-managed installs are detected and never upgraded from here — follow
# the exact-pin instructions this script prints instead.
#
# Nothing in this script hardcodes a chart version: the latest release is
# discovered from https://docs.plexicus.ai/install.sh at run time, so the
# per-release docs bump never has to touch this file.
#
# Flags:
#   --check            print the upgrade plan and exit without changing anything
#   --yes              skip the confirmation prompt (non-interactive)
#   --version X.Y.Z    upgrade to this version instead of the latest
#   --values FILE      production: values file to upgrade with
#   --registry OCI     air-gapped: internal OCI path to the chart,
#                      e.g. oci://my-internal-registry.example.com/charts/plexicus
#   --backup-dir DIR   where to write the pre-upgrade backup
#   --skip-backup      skip the automatic database backup (you accept the risk;
#                      schema migrations run during the upgrade and helm
#                      rollback does NOT revert them)
#   --air-gapped       force the air-gapped path (auto-detected otherwise)
#   --force-helm       proceed even if ArgoCD management cannot be ruled out
#   --help             this text
#
# Environment overrides (mainly for tests / unprivileged runs):
#   PLEXICUS_EVAL_STATE_DIR  evaluation state dir (default /root/plexicus-eval)
#   PLEXICUS_UPDATE_STATE    persisted-inputs file (default /root/.plexicus-update.env);
#                            setting it lifts the root requirement
# ==============================================================================
set -euo pipefail
set -E   # errtrace: the ERR trap must fire inside functions too — without it
         # a failed `helm upgrade` would exit silently and the rollback
         # guidance below would never be printed.

# -------------------------------------------------------------------- output
C_GREEN=$'\033[1;32m'; C_RED=$'\033[1;31m'; C_YELLOW=$'\033[1;33m'; C_BLUE=$'\033[1;34m'; C_OFF=$'\033[0m'
say()  { echo "${C_BLUE}==>${C_OFF} $*"; }
ok()   { echo "${C_GREEN}  [ok]${C_OFF} $*"; }
warn() { echo "${C_YELLOW}  [warn]${C_OFF} $*"; }
die()  { echo "${C_RED}ERROR:${C_OFF} $*" >&2; exit 1; }

# --------------------------------------------------------------- input helper
# Works when the script is piped into bash: prompts read from /dev/tty.
ask() { # ask <var-name> <prompt> [secret]
  local var="$1" prompt="$2" secret="${3:-}" value=""
  # Open probe, not an existence test: the /dev/tty node exists even with no
  # controlling terminal, and read then dies with ENXIO under set -e.
  if { : < /dev/tty; } 2>/dev/null; then
    if [ -n "$secret" ]; then
      read -r -s -p "$prompt: " value < /dev/tty; echo
    else
      read -r -p "$prompt: " value < /dev/tty
    fi
    printf -v "$var" '%s' "$value"
  else
    die "$var is not set and no TTY is available for prompting. Set it as an environment variable or pass the corresponding flag."
  fi
}

confirm() { # confirm <question> -> 0=yes 1=no
  local answer=""
  { : < /dev/tty; } 2>/dev/null || return 1
  read -r -p "$1 [y/N]: " answer < /dev/tty
  case "$answer" in y|Y|yes|YES) return 0 ;; *) return 1 ;; esac
}

# ack_or_die <warn-if-yes> <prompt> — a risk acknowledgement. With --yes the
# prompt is skipped (the warning still prints); interactively it is mandatory.
ack_or_die() {
  if [ "$ASSUME_YES" = 1 ]; then
    warn "$1 — continuing because --yes was passed"
    return 0
  fi
  confirm "$2" || die "aborted — nothing has been upgraded"
}

# Version comparison: ver_lt a b  ->  true when a < b (semantic sort).
ver_lt() {
  local min; min=$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -n1)
  [ "$min" = "$1" ] && [ "$1" != "$2" ]
}
# crossed <gate> -> current < gate <= target (gates are cumulative: a client
# jumping several versions runs every gate they cross).
crossed() { ver_lt "${CURRENT:-0}" "$1" && ! ver_lt "${TARGET:-0}" "$1"; }

# ------------------------------------------------------------------ defaults
DOCS_BASE="https://docs.plexicus.ai"
REGISTRY_HOST="europe-west3-docker.pkg.dev"
CHART_OCI="oci://${REGISTRY_HOST}/plexicus-registry/charts/plexicus"
NAMESPACE="plexicus"
STATE_DIR="${PLEXICUS_EVAL_STATE_DIR:-/root/plexicus-eval}"
UPDATE_STATE="${PLEXICUS_UPDATE_STATE:-/root/.plexicus-update.env}"

DO_CHECK=0 ASSUME_YES=0 SKIP_BACKUP=0 FORCED_AIRGAPPED=0 FORCE_HELM=0 TARGET_EXPLICIT=0
TARGET="${PLEXICUS_TARGET_VERSION:-}" VALUES_FILE="${PLEXICUS_VALUES_FILE:-}"
REGISTRY_OCI="${PLEXICUS_CHART_REGISTRY:-}" BACKUP_DIR=""

usage() {
  cat <<'EOF'
Plexicus on-prem update — upgrade an existing install to the latest release.

Usage: update.sh [flags]

  --check            print the upgrade plan and exit without changing anything
  --yes              skip the confirmation prompt (non-interactive)
  --version X.Y.Z    upgrade to this version instead of the latest
  --values FILE      values file to upgrade with (production / air-gapped)
  --registry OCI     air-gapped: internal OCI path to the chart
  --backup-dir DIR   where to write the pre-upgrade backup
  --skip-backup      skip the automatic database backup (you accept the risk)
  --air-gapped       force the air-gapped path (auto-detected otherwise)
  --force-helm       proceed even if ArgoCD management cannot be ruled out
  --help             this text

Docs: https://docs.plexicus.ai/docs/self-hosted/operations
EOF
  exit 0
}

while [ $# -gt 0 ]; do
  case "$1" in
    --check)        DO_CHECK=1 ;;
    --yes)          ASSUME_YES=1 ;;
    --version)      shift; TARGET="${1:-}"; [ -n "$TARGET" ] || die "--version needs a value"; TARGET_EXPLICIT=1 ;;
    --version=*)    TARGET="${1#--version=}"; [ -n "$TARGET" ] || die "--version needs a value"; TARGET_EXPLICIT=1 ;;
    --values)       shift; VALUES_FILE="${1:-}"; [ -n "$VALUES_FILE" ] || die "--values needs a path" ;;
    --values=*)     VALUES_FILE="${1#--values=}"; [ -n "$VALUES_FILE" ] || die "--values needs a path" ;;
    --registry)     shift; REGISTRY_OCI="${1:-}"; [ -n "$REGISTRY_OCI" ] || die "--registry needs an oci:// path" ;;
    --registry=*)   REGISTRY_OCI="${1#--registry=}"; [ -n "$REGISTRY_OCI" ] || die "--registry needs an oci:// path" ;;
    --backup-dir)   shift; BACKUP_DIR="${1:-}"; [ -n "$BACKUP_DIR" ] || die "--backup-dir needs a path" ;;
    --backup-dir=*) BACKUP_DIR="${1#--backup-dir=}"; [ -n "$BACKUP_DIR" ] || die "--backup-dir needs a path" ;;
    --skip-backup)  SKIP_BACKUP=1 ;;
    --air-gapped)   FORCED_AIRGAPPED=1 ;;
    --force-helm)   FORCE_HELM=1 ;;
    --help|-h)      usage ;;
    *)              die "unknown flag: $1 (try --help)" ;;
  esac
  shift
done

# Persisted per-install inputs (paths only unless noted) — this file is what
# keeps a re-run non-interactive: install.sh never persists the registry key,
# license or domain paths, so without this a re-run re-prompts and a missing
# PLEXICUS_DOMAIN silently regresses a custom domain to the default.
# Explicit flags and environment always win: the state file only fills blanks.
load_update_state() {
  [ -r "$UPDATE_STATE" ] || return 0
  local _t="$TARGET" _v="$VALUES_FILE" _r="$REGISTRY_OCI" _b="$BACKUP_DIR" \
        _d="${PLEXICUS_DOMAIN:-}" _g="${PLEXICUS_GAR_KEY_FILE:-}" \
        _l="${PLEXICUS_LICENSE_FILE:-}" _a="${PLEXICUS_AI_KEY:-}" _e="${PLEXICUS_ENV_FILE:-}"
  # shellcheck disable=SC1090
  . "$UPDATE_STATE"
  if [ -n "$_t" ]; then TARGET="$_t"; fi
  if [ -n "$_v" ]; then VALUES_FILE="$_v"; else VALUES_FILE=""; fi
  if [ -n "$_r" ]; then REGISTRY_OCI="$_r"; fi
  if [ -n "$_b" ]; then BACKUP_DIR="$_b"; fi
  if [ -n "$_d" ]; then PLEXICUS_DOMAIN="$_d"; fi
  if [ -n "$_g" ]; then PLEXICUS_GAR_KEY_FILE="$_g"; fi
  if [ -n "$_l" ]; then PLEXICUS_LICENSE_FILE="$_l"; fi
  if [ -n "$_a" ]; then PLEXICUS_AI_KEY="$_a"; fi
  if [ -n "$_e" ]; then PLEXICUS_ENV_FILE="$_e"; fi
  return 0
}
save_update_state() { # save_update_state VAR...
  local v
  {
    echo "# Written by update.sh — inputs for the next update run."
    for v in "$@"; do
      if [ -n "${!v:-}" ]; then
        printf '%s=%q\n' "$v" "${!v}"
      fi
    done
  } > "$UPDATE_STATE"
  chmod 600 "$UPDATE_STATE"
  ok "inputs saved to $UPDATE_STATE (mode 600) for future updates"
}
load_update_state

# ------------------------------------------------------------------- helpers
helm_release_exists() { helm status plexicus -n "$NAMESPACE" >/dev/null 2>&1; }

current_version() {
  helm list -n "$NAMESPACE" --filter '^plexicus$' -o json 2>/dev/null \
    | tr -d ' \n' | grep -o '"chart":"plexicus-[0-9][0-9.]*"' | head -1 \
    | sed 's/.*plexicus-//;s/"$//' || true
}

release_domain() { # best-effort: global.domain from live values or eval overrides
  local d=""
  if [ "$COHORT" = "eval" ] && [ -f "$STATE_DIR/overrides.yaml" ]; then
    d=$(awk '/^global:/{g=1} g && /^  domain:/{print $2; exit}' "$STATE_DIR/overrides.yaml")
  else
    d=$(helm get values plexicus -n "$NAMESPACE" -o yaml 2>/dev/null \
          | awk '/^global:/{g=1} g && /^  domain:/{print $2; exit}') || true
  fi
  echo "${d:-}"
}

resolve_latest() {
  if [ -n "$TARGET" ]; then
    ok "target version: $TARGET (explicit)"
    return 0
  fi
  say "Discovering the latest release"
  local body line=""
  # Fetch the whole body before parsing: `curl | grep -m1` under pipefail
  # closes the pipe at the first match, so curl takes SIGPIPE and a working
  # fetch is misreported as unreachable (field report 2026-10-14).
  if ! body=$(curl -fsSL --max-time 20 "$DOCS_BASE/install.sh" 2>/dev/null); then
    if [ "$FORCED_AIRGAPPED" = 1 ]; then
      die "cannot reach $DOCS_BASE (expected when air-gapped) — pass --version X.Y.Z"
    fi
    die "cannot reach $DOCS_BASE/install.sh to discover the latest version — pass --version X.Y.Z"
  fi
  while IFS= read -r l; do
    case "$l" in
      CHART_VERSION=*) line=$l; break ;;
    esac
  done <<< "$body"
  if [ -z "$line" ]; then
    die "no CHART_VERSION line found in $DOCS_BASE/install.sh — pass --version X.Y.Z"
  fi
  # install.sh line: CHART_VERSION="${PLEXICUS_CHART_VERSION:-<latest>}"  ->  <latest>
  TARGET=$(printf '%s' "$line" | sed 's/.*:-//; s/[^0-9.].*//')
  if [ -z "$TARGET" ]; then
    die "could not parse the latest version out of $DOCS_BASE/install.sh"
  fi
  ok "latest release: $TARGET"
}

validate_target() {
  printf '%s' "$TARGET" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \
    || die "target version '$TARGET' is not X.Y.Z"
}

# ArgoCD-managed releases must never be upgraded from here: helm would fight
# the controller on the next sync. Two independent detectors; the CRD path
# fails CLOSED when the CRDs exist but the Applications cannot be read (RBAC),
# because guessing "not managed" is how helm ends up fighting the controller.
detect_argocd() {
  ARGOCD=0
  local labels apps
  # 1) workload labels — any Application name, no CRD read needed
  labels=$(kubectl -n "$NAMESPACE" get deploy fastapi -o jsonpath='{.metadata.labels}' 2>/dev/null || true)
  case "$labels" in
    *argocd*) ARGOCD=1; return 0 ;;
  esac
  # 2) Application CRs (any name mentioning plexicus, any namespace)
  if ! kubectl get crd applications.argoproj.io >/dev/null 2>&1; then
    return 0   # ArgoCD not installed on this cluster
  fi
  if ! apps=$(kubectl get applications.argoproj.io -A -o name 2>/dev/null); then
    if [ "$FORCE_HELM" = 1 ]; then
      warn "ArgoCD CRDs exist but the Applications cannot be read (RBAC?) — continuing because --force-helm was passed"
      return 0
    fi
    die "the ArgoCD CRDs are installed but 'kubectl get applications.argoproj.io -A' failed (RBAC?) — cannot rule out ArgoCD management. Fix the permissions, or pass --force-helm if you are certain this release is not ArgoCD-managed."
  fi
  case "$apps" in
    *plexicus*|*Plexicus*) ARGOCD=1 ;;
  esac
  return 0
}

detect_cohort() {
  COHORT="" ARGOCD=0 AIRGAPPED=0
  command -v helm    >/dev/null 2>&1 || die "helm not found — install helm >= 3.12 first (see the Production Installation guide)"
  command -v kubectl >/dev/null 2>&1 || die "kubectl not found — install kubectl >= 1.25 first"

  # Evaluation state first: helm does not read k3s's kubeconfig on its own, so
  # without this pin every helm query below runs against whatever context the
  # operator's ambient kubeconfig points at — while install.sh (which does pin
  # KUBECONFIG itself) would mutate the LOCAL machine. Split brain, silently.
  if [ -d "$STATE_DIR" ] && [ -f "$STATE_DIR/credentials.env" ]; then
    COHORT="eval"
    if [ -n "${KUBECONFIG:-}" ]; then
      warn "evaluation state present but KUBECONFIG is preset ($KUBECONFIG) — it must point at THIS machine's cluster, or the plan and the installer will target different machines"
    elif [ -r /etc/rancher/k3s/k3s.yaml ]; then
      export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
      ok "using the local k3s kubeconfig"
    else
      die "evaluation state found at $STATE_DIR, but /etc/rancher/k3s/k3s.yaml is missing — this is not the evaluator machine. Move $STATE_DIR aside to upgrade a production install from this host."
    fi
  else
    COHORT="prod"
  fi

  if ! kubectl cluster-info >/dev/null 2>&1; then
    if [ "$DO_CHECK" = 1 ]; then
      COHORT="none"
      return 0
    fi
    die "kubectl cannot reach the cluster (KUBECONFIG=${KUBECONFIG:-unset}) — check your kubeconfig / context"
  fi

  if ! helm_release_exists; then
    if [ "$DO_CHECK" = 1 ]; then
      COHORT="none"
      return 0
    fi
    die "no Plexicus release found (helm status plexicus -n $NAMESPACE failed) — this tool upgrades existing installs only"
  fi

  detect_argocd

  # Air-gapped: values point at a customer registry instead of ours, or the
  # operator says so explicitly.
  local values
  values=$(helm get values plexicus -n "$NAMESPACE" -o yaml 2>/dev/null || true)
  if [ "$FORCED_AIRGAPPED" = 1 ]; then
    AIRGAPPED=1
  elif printf '%s' "$values" | grep -q 'imageRegistry:'; then
    if ! printf '%s' "$values" | grep -qF "$REGISTRY_HOST"; then
      AIRGAPPED=1
    fi
  fi
  if [ "$AIRGAPPED" = 1 ]; then
    COHORT="air-gapped"
  fi
  return 0
}

print_argocd_bail() {
  warn "This install is managed by ArgoCD — update.sh will not run helm against it."
  say "Upgrade with the exact-pin flow instead:"
  cat <<'EOF'
  1. Edit your ArgoCD application's targetRevision to the new version:
       kubectl get applications -A                       # find the plexicus app
       kubectl -n <app-namespace> get application <name> -o yaml
  2. Apply the change (edit argocd/application.yaml if you keep one in Git):
       kubectl apply -n <app-namespace> -f argocd/application.yaml
  3. Watch the sync:
       argocd app get <name>
EOF
}

preflight() {
  say "Preflight"
  # --check only reads state; a real run mutates the system and needs root
  # (default paths live under /root and /var/backups, install.sh requires it).
  if [ "$DO_CHECK" != 1 ] && [ -z "${PLEXICUS_UPDATE_STATE:-}" ]; then
    [ "$(id -u)" -eq 0 ] || die "Run as root (sudo) — or set PLEXICUS_UPDATE_STATE to a writable path to run unprivileged."
  fi
  if [ "$COHORT" = "eval" ]; then
    local disk_gb; disk_gb=$(df -BG --output=avail / | tail -1 | tr -dc '0-9')
    if [ -n "$disk_gb" ] && [ "$disk_gb" -lt 40 ]; then
      warn "install.sh requires >= 40 GB free on / (found ${disk_gb} GB) — its preflight may stop the upgrade; grow the disk first if so"
    fi
  fi
  ok "preflight ok"
}

compute_gates() {
  # Populates GATES (pre-upgrade) and POST_GATES (after the upgrade) as
  # newline-separated human-readable lines. Cumulative across old→new jumps.
  GATES="" POST_GATES="" NOTES=""
  if crossed 1.2.59; then
    GATES+="delete any registry-sync-* Temporal schedules (required by 1.2.59 before the worker drops those workflows)"$'\n'
    NOTES+="1.2.59: bring-your-own-AI connection writes now need an on-prem licence or AI_BYOAI_ENABLED=true."$'\n'
  fi
  if crossed 1.2.50; then
    POST_GATES+="delete the plexicus-analysis-scheduler Secret if present (1.2.50 removed the analysis-scheduler component)"$'\n'
    if [ "$COHORT" != "eval" ]; then
      NOTES+="1.2.50: drop any 'analysis-scheduler:' block from your values file (the chart ignores it)."$'\n'
    fi
  fi
  if crossed 1.2.57; then
    NOTES+="1.2.57: PlexiFlow approval gates without approver_role now allow admins only; a nightly xbom-daily-refresh schedule is created automatically."$'\n'
  fi
  # websocketDb: only the client-managed values file can pin it; eval
  # overrides are regenerated by the installer on every run.
  if [ "$COHORT" != "eval" ] && [ "$COHORT" != "none" ]; then
    local wdb values
    values=$(helm get values plexicus -n "$NAMESPACE" -o yaml 2>/dev/null || true)
    wdb=$(printf '%s\n' "$values" | grep -m1 'websocketDb:' | awk '{print $2}' || true)
    if [ -n "$wdb" ] && [ "$wdb" != "2" ]; then
      WEBSOCKET_DB_BAD="$wdb"
    fi
    if printf '%s' "$values" | grep -q 'registryConnections:'; then
      NOTES+="your values still carry global.features.registryConnections (removed in 1.2.59 — harmless, but clean it up)."$'\n'
    fi
  fi
  return 0
}

backup_pre_upgrade() {
  BACKUP_DIR=${BACKUP_DIR:-/var/backups/plexicus-$(date -u +%Y%m%dT%H%M%SZ)}
  say "Backing up to $BACKUP_DIR"
  mkdir -p "$BACKUP_DIR" || die "cannot create backup dir $BACKUP_DIR — nothing has been upgraded"

  # Helm state: revision history for rollback reference, and the live values
  # (the only copy of user-supplied values if my-values.yaml went missing).
  # helm-values.yaml is written via a temp file so a failed `helm get values`
  # can never leave a 0-byte file that the later resolver would silently pass
  # to `helm upgrade -f` (which would drop every override, including
  # global.imageRegistry, and re-resolve images from the public defaults).
  helm history plexicus -n "$NAMESPACE" | tail -1 > "$BACKUP_DIR/helm-history.txt" 2>/dev/null || true
  if helm get values plexicus -n "$NAMESPACE" -o yaml > "$BACKUP_DIR/.helm-values.tmp" 2>/dev/null; then
    mv "$BACKUP_DIR/.helm-values.tmp" "$BACKUP_DIR/helm-values.yaml"
  else
    rm -f "$BACKUP_DIR/.helm-values.tmp"
    warn "could not capture live helm values into the backup"
  fi

  # Databases: the bundled MongoDB/PostgreSQL are scriptable; external ones
  # are the operator's own and can only be their responsibility.
  if ! kubectl -n "$NAMESPACE" get deploy mongodb >/dev/null 2>&1; then
    warn "external database detected (no bundled mongodb deployment)"
    cat <<'EOF'
  Take a backup with your own tooling BEFORE continuing — schema migrations
  may run during the upgrade and `helm rollback` does NOT revert them.
  The reference procedure is in the Backup and Restore guide:
    https://docs.plexicus.ai/docs/self-hosted/backup-restore
EOF
    if [ "$SKIP_BACKUP" = 1 ]; then
      warn "continuing without a backup (--skip-backup)"
      return 0
    fi
    ack_or_die "continuing without an automatic backup" \
      "Backup taken (or accepted risk) — continue without an automatic backup?"
    return 0
  fi

  local pw mpod
  if ! pw=$(kubectl -n "$NAMESPACE" get secret mongodb -o jsonpath='{.data.mongodb-root-password}' 2>/dev/null | base64 -d) \
     || [ -z "$pw" ]; then
    warn "could not read the bundled MongoDB root password from Secret/mongodb (custom auth.existingSecret?)"
    cat <<'EOF'
  Dump MongoDB manually with your own credentials BEFORE continuing — schema
  migrations may run during the upgrade and `helm rollback` does NOT revert
  them. The reference procedure is in the Backup and Restore guide:
    https://docs.plexicus.ai/docs/self-hosted/backup-restore
EOF
    if [ "$SKIP_BACKUP" = 1 ]; then
      warn "continuing without a backup (--skip-backup)"
      return 0
    fi
    ack_or_die "continuing without an automatic MongoDB backup" \
      "MongoDB backup taken (or accepted risk) — continue?"
    return 0
  fi
  kubectl -n "$NAMESPACE" exec deploy/mongodb -- \
    mongodump --host localhost:27017 --username root --password "$pw" \
    --authenticationDatabase admin --db plexicus \
    --gzip --archive=/tmp/pre-upgrade.archive >/dev/null \
    || die "mongodump failed — nothing has been upgraded; see the Backup and Restore guide"
  mpod=$(kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=mongodb -o jsonpath='{.items[0].metadata.name}' || true)
  if [ -z "$mpod" ]; then
    die "could not find the mongodb pod to copy the archive from — nothing has been upgraded; see the Backup and Restore guide"
  fi
  kubectl -n "$NAMESPACE" cp "${mpod}:/tmp/pre-upgrade.archive" "$BACKUP_DIR/mongo.archive" \
    || die "copying the MongoDB archive out of the pod failed — nothing has been upgraded"
  kubectl -n "$NAMESPACE" exec "$mpod" -- rm -f /tmp/pre-upgrade.archive >/dev/null 2>&1 || true
  ok "MongoDB archived to $BACKUP_DIR/mongo.archive"

  if kubectl -n "$NAMESPACE" get pod temporal-postgresql-0 >/dev/null 2>&1; then
    # With custom POSTGRES_USER=temporal the superuser password exists only in
    # POSTGRES_POSTGRES_PASSWORD_FILE; a non-TTY exec cannot prompt for it.
    kubectl -n "$NAMESPACE" exec temporal-postgresql-0 -- \
      bash -c 'PGPASSWORD="$(cat "$POSTGRES_POSTGRES_PASSWORD_FILE")" exec pg_dump -U postgres -Fc temporal' \
      > "$BACKUP_DIR/temporal.dump" \
      || die "pg_dump failed — nothing has been upgraded; see the Backup and Restore guide"
    ok "PostgreSQL dumped to $BACKUP_DIR/temporal.dump"
  else
    warn "temporal-postgresql-0 not found (external database) — dump it with your own tooling"
  fi
  warn "object storage snapshot is provider-specific — see the Backup and Restore guide if you rely on uploaded artifacts"
  ok "backup complete — keep $BACKUP_DIR until the upgrade is verified"
}

# One-off admin-tools pod — used when the temporal-admintools Deployment is
# disabled (evaluation installs trim it) but the registry-sync gate still
# needs the temporal CLI. TEMPORAL_ADDRESS matches the chart's in-cluster
# service name (global.required temporal host).
_admin_tools_pod() { # _admin_tools_pod <temporal CLI args...>; stdout on success
  local rname="plexicus-schedule-cli"
  # Same rewrite the chart applies to its temporal-setup Job
  # (worker/templates/temporal-setup-job.yaml): global.imageRegistry, or
  # docker.io when unset/empty. Hardcoding docker.io here would break
  # air-gapped installs where only the mirror registry is reachable.
  local reg="" img out
  reg=$(helm get values plexicus -n "$NAMESPACE" -o yaml 2>/dev/null \
        | awk '/^global:/{g=1} g && /^  imageRegistry:/{print $2; exit}' || true)
  reg=${reg%\"}; reg=${reg#\"}
  reg=${reg%\'}; reg=${reg#\'}
  # Helm's `default` treats "" and null as unset — mirror that here.
  if [ -z "$reg" ] || [ "$reg" = "null" ]; then reg="docker.io"; fi
  img="$reg/temporalio/admin-tools:1.29.1-tctl-1.18.4-cli-1.5.0"
  kubectl -n "$NAMESPACE" delete pod "$rname" --ignore-not-found --wait=true >/dev/null 2>&1 || true
  kubectl -n "$NAMESPACE" run "$rname" --restart=Never --image="$img" \
    --env=TEMPORAL_ADDRESS=temporal-frontend:7233 -- \
    temporal "$@" >/dev/null 2>&1 || return 1
  # 240s: cold nodes may need to pull admin-tools (~800MB) first.
  if ! kubectl -n "$NAMESPACE" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$rname" --timeout=240s >/dev/null 2>&1; then
    kubectl -n "$NAMESPACE" delete pod "$rname" --ignore-not-found --wait=false >/dev/null 2>&1 || true
    return 1
  fi
  out=$(kubectl -n "$NAMESPACE" logs "$rname" 2>/dev/null || true)
  kubectl -n "$NAMESPACE" delete pod "$rname" --ignore-not-found --wait=false >/dev/null 2>&1 || true
  printf '%s\n' "$out"
  return 0
}

gate_pre_registry_sync() {
  crossed 1.2.59 || return 0
  say "Pre-upgrade gate: registry-sync-* Temporal schedules"
  local list="" list_ok=0 ids id prev via_pod=0

  if kubectl -n "$NAMESPACE" get deploy temporal-admintools >/dev/null 2>&1; then
    prev=$(kubectl -n "$NAMESPACE" get deploy temporal-admintools -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 0)
    kubectl -n "$NAMESPACE" scale deploy/temporal-admintools --replicas=1 >/dev/null \
      || { warn "could not scale temporal-admintools to 1"; return 1; }
    kubectl -n "$NAMESPACE" rollout status deploy/temporal-admintools --timeout=120s >/dev/null 2>&1 || true
    if list=$(kubectl -n "$NAMESPACE" exec deploy/temporal-admintools -- temporal schedule list 2>&1); then
      list_ok=1
    else
      warn "'temporal schedule list' failed: ${list}"
      list=""
    fi
    kubectl -n "$NAMESPACE" scale deploy/temporal-admintools --replicas="${prev:-0}" >/dev/null 2>&1 || true
  else
    via_pod=1
    say "temporal-admintools is not deployed — listing schedules with a one-off admin-tools pod"
    if list=$(_admin_tools_pod schedule list); then
      list_ok=1
    else
      list=""
    fi
  fi

  if [ "$list_ok" != 1 ]; then
    ack_or_die "could not verify that no registry-sync-* schedules exist (stale registry-sync-* schedules will error after the upgrade)" \
      "Could not list Temporal schedules — verify manually that no registry-sync-* schedules exist ('temporal schedule list'), then continue"
    return 0
  fi
  ids=$(printf '%s\n' "$list" | grep -oE 'registry-sync-[A-Za-z0-9_.-]+' | sort -u || true)
  if [ -z "$ids" ]; then
    ok "no registry-sync-* schedules — nothing to do"
    return 0
  fi
  while IFS= read -r id; do
    if [ -z "$id" ]; then
      continue
    fi
    local delete_ok=1
    if [ "$via_pod" = 1 ]; then
      _admin_tools_pod schedule delete --schedule-id "$id" >/dev/null 2>&1 || delete_ok=0
    else
      kubectl -n "$NAMESPACE" exec deploy/temporal-admintools -- temporal schedule delete --schedule-id "$id" >/dev/null 2>&1 || delete_ok=0
    fi
    if [ "$delete_ok" = 1 ]; then
      ok "deleted schedule $id"
    else
      ack_or_die "could not delete schedule $id" \
        "Could not delete schedule $id — delete it manually ('temporal schedule delete --schedule-id $id'), then continue"
    fi
  done <<< "$ids"
  return 0
}

gate_post_analysis_scheduler() {
  crossed 1.2.50 || return 0
  if kubectl -n "$NAMESPACE" get secret plexicus-analysis-scheduler >/dev/null 2>&1; then
    if ! kubectl -n "$NAMESPACE" delete secret plexicus-analysis-scheduler; then
      warn "could not delete the retired Secret — remove it yourself: kubectl -n $NAMESPACE delete secret plexicus-analysis-scheduler"
    else
      ok "removed the retired plexicus-analysis-scheduler Secret"
    fi
  fi
  return 0
}

run_eval_upgrade() {
  say "Upgrading via the installer (evaluation install)"
  local install_sh="$STATE_DIR/install.sh-$TARGET"
  curl -fsSL "$DOCS_BASE/install.sh" -o "$install_sh" \
    || die "could not download $DOCS_BASE/install.sh — retry when the network is available"
  # install.sh regenerates overrides.yaml from these: without them a re-run
  # re-prompts for the three delivery inputs and resets a custom domain to the
  # default, rewriting TLS for the wrong host. export happens unconditionally —
  # a state-loaded (non-exported) PLEXICUS_DOMAIN would otherwise never reach
  # the installer subprocess.
  PLEXICUS_DOMAIN="${PLEXICUS_DOMAIN:-${EVAL_DOMAIN:-}}"
  export PLEXICUS_CHART_VERSION="$TARGET"
  if [ -n "$PLEXICUS_DOMAIN" ]; then
    export PLEXICUS_DOMAIN
  fi
  if [ -n "${PLEXICUS_GAR_KEY_FILE:-}" ]; then export PLEXICUS_GAR_KEY_FILE; fi
  if [ -n "${PLEXICUS_LICENSE_FILE:-}" ]; then export PLEXICUS_LICENSE_FILE; fi
  if [ -n "${PLEXICUS_AI_KEY:-}" ]; then export PLEXICUS_AI_KEY; fi
  if [ -n "${PLEXICUS_ENV_FILE:-}" ]; then export PLEXICUS_ENV_FILE; fi
  bash "$install_sh"
}

ensure_prod_registry_login() {
  helm show chart "$CHART_OCI" --version "$TARGET" >/dev/null 2>&1 && return 0
  say "Registry login required for the chart pull"
  [ -n "${PLEXICUS_GAR_KEY_FILE:-}" ] || ask PLEXICUS_GAR_KEY_FILE "Path to the registry service-account JSON key (keys.json)"
  [ -r "$PLEXICUS_GAR_KEY_FILE" ] || die "registry key not readable: $PLEXICUS_GAR_KEY_FILE"
  printf '%s' "$(cat "$PLEXICUS_GAR_KEY_FILE")" | helm registry login "$REGISTRY_HOST" \
    --username _json_key --password-stdin >/dev/null \
    || die "helm registry login failed for $REGISTRY_HOST"
  ok "logged in to $REGISTRY_HOST"
  export PLEXICUS_GAR_KEY_FILE
  return 0
}

run_prod_upgrade() {
  say "Upgrading via helm (production install)"
  if [ -z "$VALUES_FILE" ]; then
    die "internal: no values file resolved — pass --values FILE (this should have been caught while planning)"
  fi
  if [ "$AIRGAPPED" = 1 ]; then
    [ -n "$REGISTRY_OCI" ] || die "air-gapped upgrades need --registry oci://<internal-registry>/charts/plexicus"
    if ! helm show chart "$REGISTRY_OCI" --version "$TARGET" >/dev/null 2>&1; then
      die "chart not reachable at $REGISTRY_OCI --version $TARGET. Mirror the new release first (push the chart into your internal registry), per the Air-Gapped guide: $DOCS_BASE/docs/self-hosted/air-gapped"
    fi
    ok "chart $TARGET found in the internal registry"
    helm upgrade plexicus "$REGISTRY_OCI" --version "$TARGET" -n "$NAMESPACE" \
      -f "$VALUES_FILE" --timeout 20m
  else
    ensure_prod_registry_login
    helm upgrade plexicus "$CHART_OCI" --version "$TARGET" -n "$NAMESPACE" \
      -f "$VALUES_FILE" --timeout 20m
  fi
  ok "helm upgrade finished"
}

# Armed with `trap ... ERR` ONLY around the execute phase, and it must
# disarm itself first so a failing command inside the handler cannot recurse.
on_execute_error() {
  trap - ERR
  echo "" >&2
  warn "the upgrade hit an error — current state:"
  helm history plexicus -n "$NAMESPACE" 2>/dev/null | tail -5 >&2 || true
  cat >&2 <<EOF
Rollback (chart state only — schema migrations are NOT reverted):
  helm rollback plexicus -n $NAMESPACE
If data looks wrong after a rollback, restore from: ${BACKUP_DIR:-the backup you took}
Manual verification:
  kubectl -n $NAMESPACE get pods
  helm list -n $NAMESPACE
EOF
}

verify_upgrade() {
  say "Verifying"
  local got; got=$(current_version)
  [ "$got" = "$TARGET" ] || die "release reports chart $got but the target is $TARGET — inspect: helm history plexicus -n $NAMESPACE"
  ok "helm release is at $TARGET"

  local d
  for d in fastapi frontend worker; do
    kubectl -n "$NAMESPACE" rollout status "deploy/$d" --timeout=20m >/dev/null \
      || die "deploy/$d did not become ready — inspect: kubectl -n $NAMESPACE describe pod -l app.kubernetes.io/name=$d"
  done
  ok "fastapi, frontend and worker are Running"

  if [ -z "$DOMAIN" ]; then
    warn "domain unknown — run the health checks yourself: curl -k https://api.<your-domain>/health"
    return 0
  fi
  # Status-code probe only (-k): self-signed and private-CA frontends are both
  # supported install flavors, and readiness is what we are checking here.
  # DNS first (works from any machine that can resolve the domain). The
  # localhost --resolve fallback fires ONLY on evaluation installs, where the
  # name legitimately resolves on the node alone; using it elsewhere would let
  # an unrelated local :443 answer 200 and turn a broken production upgrade
  # into a false pass. `|| true` + ${code:-000}: curl already prints 000 on
  # failure, so `|| echo 000` used to concatenate it into "000000".
  local code="" deadline=$(( SECONDS + 180 ))
  while [ "$SECONDS" -lt "$deadline" ]; do
    code=$(curl -sk -o /dev/null -w "%{http_code}" --max-time 10 \
             "https://api.${DOMAIN}/health" || true)
    code=${code:-000}
    if [ "$code" != "200" ] && [ "$COHORT" = "eval" ]; then
      code=$(curl -sk -o /dev/null -w "%{http_code}" --max-time 10 \
               --resolve "api.${DOMAIN}:443:127.0.0.1" "https://api.${DOMAIN}/health" || true)
      code=${code:-000}
    fi
    if [ "$code" = "200" ]; then
      break
    fi
    sleep 5
  done
  if [ "$code" != "200" ]; then
    die "API /health never returned 200 (last: $code) — check: kubectl -n $NAMESPACE logs deploy/fastapi"
  fi
  ok "api/health 200 for api.${DOMAIN}"
}

# ============================================================== plan + run
say "Plexicus on-prem update"
resolve_latest
validate_target
detect_cohort

if [ "$ARGOCD" = 1 ]; then
  print_argocd_bail
  exit 0
fi

if [ "$COHORT" = "none" ]; then
  ok "no install found — nothing to update"
  echo "Target release: $TARGET"
  exit 0
fi

CURRENT=$(current_version)
if [ -z "$CURRENT" ]; then
  die "could not read the installed chart version (helm list -n $NAMESPACE failed)"
fi
if [ "$CURRENT" = "$TARGET" ]; then
  ok "already at $TARGET — nothing to do"
  exit 0
fi
if ! ver_lt "$CURRENT" "$TARGET"; then
  if [ "$TARGET_EXPLICIT" = 1 ]; then
    die "installed version $CURRENT is newer than the requested target $TARGET — downgrades are not supported (see Rollback in the Operations guide)"
  fi
  warn "installed version $CURRENT is newer than the published latest $TARGET — nothing to do"
  exit 0
fi

DOMAIN=$(release_domain)
if [ "$COHORT" = "eval" ]; then
  EVAL_DOMAIN="${PLEXICUS_DOMAIN:-$DOMAIN}"
else
  # Values discovery for both online and air-gapped: the operator's overlay
  # on disk, wherever it usually lives.
  if [ -z "$VALUES_FILE" ]; then
    for c in "$PWD/my-values.yaml" "$PWD/values.yaml" "$PWD/values-override.yaml" \
             "$HOME/plexicus-deploy/my-values.yaml" "$HOME/my-values.yaml"; do
      if [ -f "$c" ]; then
        VALUES_FILE="$c"
        break
      fi
    done
  fi
fi

preflight
compute_gates

# Air-gapped: prove the mirrored chart exists BEFORE anything is backed up or
# mutated — a missing chart means the mirror step is still pending.
if [ "$AIRGAPPED" = 1 ]; then
  if [ -z "$REGISTRY_OCI" ]; then
    die "air-gapped upgrades need --registry oci://<internal-registry>/charts/plexicus"
  fi
  if ! helm show chart "$REGISTRY_OCI" --version "$TARGET" >/dev/null 2>&1; then
    die "chart not reachable at $REGISTRY_OCI --version $TARGET — mirror the new release first (push the chart .tgz into your internal registry); if it is already mirrored, log in first ('helm registry login <registry-host>') and check --registry / --version. Air-Gapped guide: $DOCS_BASE/docs/self-hosted/air-gapped"
  fi
  ok "chart $TARGET reachable in the internal registry"
fi

say "Plan"
echo "  install:    $COHORT"
if [ "$AIRGAPPED" = 1 ]; then
  echo "              (air-gapped)"
fi
echo "  version:    $CURRENT  ->  $TARGET"
if [ "$COHORT" = "eval" ]; then
  echo "  method:    re-run install.sh with your saved inputs (idempotent)"
elif [ "$AIRGAPPED" = 1 ]; then
  echo "  method:    helm upgrade from $REGISTRY_OCI with ${VALUES_FILE:-<live values captured during backup>}"
else
  echo "  method:    helm upgrade with ${VALUES_FILE:-<live values captured during backup>}"
fi
if [ "$SKIP_BACKUP" = 1 ]; then
  echo "  backup:    SKIPPED (--skip-backup) — schema migrations are NOT reverted by helm rollback"
else
  echo "  backup:    ${BACKUP_DIR:-/var/backups/plexicus-<timestamp>} (mongodb + postgres + helm state)"
fi
if [ -n "$GATES" ]; then
  echo "  pre-upgrade gates:"
  printf '%s' "$GATES" | sed 's/^/    - /'
fi
if [ -n "$POST_GATES" ]; then
  echo "  post-upgrade gates:"
  printf '%s' "$POST_GATES" | sed 's/^/    - /'
fi
if [ -n "$NOTES" ]; then
  echo "  notes:"
  printf '%s' "$NOTES" | sed 's/^/    - /'
fi

if [ "$COHORT" != "eval" ]; then
  if [ -z "$VALUES_FILE" ] && [ "$SKIP_BACKUP" = 1 ]; then
    die "no values file found and --skip-backup was passed — pass --values FILE (your overlay); without a backup there are no live values to fall back to"
  fi
  if [ -n "${WEBSOCKET_DB_BAD:-}" ]; then
    die "global.required.redis.websocketDb is pinned to ${WEBSOCKET_DB_BAD} in your live values; it must be 2 or live scan progress goes quiet with no error. Edit your values file, set it to 2 (or remove the pin), then re-run."
  fi
fi

if [ "$DO_CHECK" = 1 ]; then
  ok "check only — no changes made"
  exit 0
fi

if [ "$ASSUME_YES" != 1 ]; then
  confirm "Proceed with the upgrade?" || die "aborted — nothing was changed"
fi

# Persist inputs before anything risky runs: a run that dies mid-upgrade must
# still leave the next run able to start without re-supplying every variable
# (field report 2026-10-14: the state file was never written because the save
# sat after the execute phase).
if [ "$COHORT" = "eval" ]; then
  PLEXICUS_DOMAIN="${PLEXICUS_DOMAIN:-${EVAL_DOMAIN:-}}"
  save_update_state PLEXICUS_GAR_KEY_FILE PLEXICUS_LICENSE_FILE PLEXICUS_AI_KEY PLEXICUS_ENV_FILE PLEXICUS_DOMAIN || true
fi

if [ "$SKIP_BACKUP" != 1 ]; then
  backup_pre_upgrade
fi

# Resolve the values file after the backup has captured the live copy
# (the fallback copy is what air-gapped installs without an on-disk overlay use:
# their live values carry global.imageRegistry).
if [ "$COHORT" != "eval" ]; then
  if [ -z "$VALUES_FILE" ]; then
    # -s, not -r: a 0-byte capture must not reach `helm upgrade -f` — an empty
    # values file silently drops every override (imageRegistry, domains, …).
    if [ -s "${BACKUP_DIR:-/nonexistent}/helm-values.yaml" ]; then
      VALUES_FILE="$BACKUP_DIR/helm-values.yaml"
      warn "using the live helm values captured at $VALUES_FILE — prefer passing your own with --values my-values.yaml"
    else
      die "no values file found — pass --values FILE (your overlay)"
    fi
  else
    [ -r "$VALUES_FILE" ] || die "values file not readable: $VALUES_FILE"
    ok "values file: $VALUES_FILE"
  fi
fi
if [ "$AIRGAPPED" = 1 ]; then
  save_update_state REGISTRY_OCI || true
fi

if ! gate_pre_registry_sync; then
  die "pre-upgrade gate failed — nothing has been upgraded"
fi

# Execute phase: ERR trap (with errtrace) prints the rollback guidance on any
# failure inside the functions below, then errexit stops the script.
trap on_execute_error ERR
if [ "$COHORT" = "eval" ]; then
  run_eval_upgrade
else
  run_prod_upgrade
fi
trap - ERR

gate_post_analysis_scheduler
verify_upgrade

echo ""
ok "Plexicus updated: $CURRENT -> $TARGET"
if [ -n "${BACKUP_DIR:-}" ] && [ "$SKIP_BACKUP" != 1 ]; then
  echo "  Backup: $BACKUP_DIR"
fi
echo "  If something looks wrong: helm rollback plexicus -n $NAMESPACE (chart state only; data needs the backup)"