panel/panelctl.sh

#!/usr/bin/env bash
set -euo pipefail

BASE_DIR="${PANEL_BASE_DIR:-/var/lib/containers}"
STACKS_DIR="${BASE_DIR}/stacks"
VOLUMES_DIR="${BASE_DIR}/volumes"
ROUTES_DIR="${BASE_DIR}/routes"
STATE_DIR="${BASE_DIR}/state"
APPS_DIR="${STATE_DIR}/apps"
ENV_DIR="${STATE_DIR}/env"
BACKUPS_DIR="${BASE_DIR}/backups"
# Owner of generated files that Caddy has to read (via the shared group).
PANEL_USER="${PANEL_USER:-reudy}"
PANEL_GROUP="${PANEL_GROUP:-panelroutes}"

# Set by load_app: compose file arguments, and the app's environment variables
# as KEY=VALUE words (passed to compose via env(1), never sourced).
CURRENT_APP=""
COMPOSE_ARGS=()
ORPHAN_ARGS=()
APP_ENV_ARGS=()
APP_ENV_KEYS=()

FORWARD_AUTH_BLOCK='  forward_auth 127.0.0.1:9091 {
    uri /api/authz/forward-auth
    copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
  }
'

validate_route_entry() {
  local entry="$1"
  # Format: domain|upstream[|path[|service:port]]. The optional last field names
  # the compose service and container port that upstream (a host port the panel
  # picked) is published from; see prepare_ports_override.
  local domain upstream path target
  IFS='|' read -r domain upstream path target <<< "${entry}"
  [[ -n "${domain}" ]] || fail "empty domain in route entry '${entry}'"
  [[ -n "${upstream}" ]] || fail "empty upstream in route entry '${entry}'"
  validate_single_domain "${domain}"
  # Validate upstream has a port
  local upstream_port="${upstream##*:}"
  [[ "${upstream_port}" =~ ^[0-9]+$ ]] || fail "upstream '${upstream}' missing numeric port in route entry '${entry}'"
  validate_port "${upstream_port}"
  if [[ -n "${path}" ]]; then
    [[ "${path}" == /* ]] || fail "path '${path}' must start with / in route entry '${entry}'"
  fi
  if [[ -n "${target}" ]]; then
    [[ "${target}" =~ ^([A-Za-z0-9._-]*):([0-9]+)$ ]] || fail "target '${target}' must look like service:port in route entry '${entry}'"
    (( BASH_REMATCH[2] >= 1 && BASH_REMATCH[2] <= 65535 )) || fail "container port in '${target}' must be between 1 and 65535"
    [[ "${upstream}" =~ ^127\.0\.0\.1:[0-9]+$ ]] || fail "a route to a service must use a 127.0.0.1 upstream in route entry '${entry}'"
  fi
}

validate_routes() {
  local routes_str="$1"
  IFS=',' read -ra entries <<< "${routes_str}"
  [[ ${#entries[@]} -ge 1 ]] || fail "at least one route is required"
  for entry in "${entries[@]}"; do
    entry="$(echo "${entry}" | xargs)"
    validate_route_entry "${entry}"
  done
}

usage() {
  cat <<'EOF'
panelctl - minimal app panel helper

Usage:
  panelctl init <name> "<domain>|<upstream>[,...]" [auth]
  panelctl set-routes <name> "<domain>|<upstream>[,...]"
  panelctl render-route <name>
  panelctl deploy <name>
  panelctl restart <name>
  panelctl stop <name>
  panelctl status <name>
  panelctl logs <name> [--tail N] [--follow] [--service S]
  panelctl services <name>
  panelctl containers
  panelctl stats
  panelctl exec <name> <container>
  panelctl remove <name> [--keep-volumes]
  panelctl backup <name>
  panelctl list-backups <name>
  panelctl restore <name> <backup-file>
  panelctl volume-clear <name>
  panelctl validate-compose <name>
  panelctl list
  panelctl show <name>

Each route is a domain|upstream pair. Upstream is host:port. An optional path
and service:port target follow: domain|127.0.0.1:18090||web:8080 publishes
container port 8080 of service "web" on 127.0.0.1:18090 when the app deploys.
Multiple routes are comma-separated:
  panelctl init myapp "app.example.com|127.0.0.1:18080,api.example.com|127.0.0.1:18081" true

Wildcard domains are supported (requires DNS challenge in Caddy):
  panelctl init myapp "*.example.com|127.0.0.1:18080" true

Examples:
  panelctl init whoami "whoami.reudy.net|127.0.0.1:18080" true
  panelctl deploy whoami
  panelctl restart whoami
  panelctl status whoami
  panelctl logs whoami --tail 50
  panelctl backup whoami
  panelctl list-backups whoami
  panelctl restore whoami whoami-20260101-120000.zip
EOF
}

fail() {
  echo "error: $*" >&2
  exit 1
}

log() {
  local level="${1:-info}"
  local msg="${2:-}"
  echo "${msg}" | systemd-cat -t panelctl -p "${level}" 2>/dev/null || true
}

# Forward stdin to the journal (or drop it where there is none).
journal_copy() {
  systemd-cat -t panelctl -p info 2>/dev/null || cat >/dev/null
}

ensure_base_dirs() {
  mkdir -p "${STACKS_DIR}" "${VOLUMES_DIR}" "${ROUTES_DIR}" "${APPS_DIR}" "${BACKUPS_DIR}"
}

validate_name() {
  local name="$1"
  [[ "${name}" =~ ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ ]] || fail "invalid name '${name}' (use lowercase slug)"
}

validate_single_domain() {
  local domain="$1"
  # Allow wildcard prefix *.
  local check="${domain}"
  if [[ "${check}" == \*.* ]]; then
    check="${check#\*.}"
  fi
  [[ "${check}" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]] || fail "invalid domain '${domain}'"
  [[ "${domain}" == *.* ]] || fail "domain '${domain}' must include a dot"
}

validate_domains() {
  local domains_str="$1"
  IFS=',' read -ra domains <<< "${domains_str}"
  [[ ${#domains[@]} -ge 1 ]] || fail "at least one domain is required"
  for d in "${domains[@]}"; do
    d="$(echo "${d}" | xargs)" # trim whitespace
    validate_single_domain "${d}"
  done
}

validate_port() {
  local port="$1"
  [[ "${port}" =~ ^[0-9]+$ ]] || fail "port must be numeric"
  (( port >= 1024 && port <= 65535 )) || fail "port must be in range 1024-65535"
}

app_manifest() {
  local name="$1"
  echo "${APPS_DIR}/${name}.env"
}

app_stack_dir() {
  local name="$1"
  echo "${STACKS_DIR}/${name}"
}

app_volume_dir() {
  local name="$1"
  echo "${VOLUMES_DIR}/${name}"
}

# All routes go into a single aggregate file that Caddy imports.
app_route_file() {
  echo "${ROUTES_DIR}/routes.caddy"
}

# The routes file is shared by all apps and rewritten read-modify-write, so
# concurrent panelctl runs (the API handles requests in parallel) must take turns.
routes_lock() {
  exec 9>"${ROUTES_DIR}/.routes.lock"
  if command -v flock >/dev/null 2>&1; then
    flock -w 30 9 || fail "timed out waiting for the routes file lock"
  fi
}

routes_unlock() {
  exec 9>&-
}

load_app() {
  local name="$1"
  CURRENT_APP="${name}"
  local manifest
  manifest="$(app_manifest "${name}")"
  [[ -f "${manifest}" ]] || fail "app '${name}' does not exist"
  unset APP_COMPOSE_PROJECT
  # shellcheck disable=SC1090
  source "${manifest}"

  # Backward compat: migrate old APP_DOMAIN/APP_PORT/APP_UPSTREAM to APP_ROUTES
  if [[ -z "${APP_ROUTES:-}" && -n "${APP_DOMAIN:-}" ]]; then
    local upstream="${APP_UPSTREAM:-127.0.0.1:${APP_PORT:-18080}}"
    local routes=""
    local domains_str="${APP_DOMAINS:-${APP_DOMAIN}}"
    IFS=',' read -ra domain_arr <<< "${domains_str}"
    for d in "${domain_arr[@]}"; do
      d="$(echo "${d}" | xargs)"
      if [[ -n "${routes}" ]]; then
        routes="${routes},${d}|${upstream}"
      else
        routes="${d}|${upstream}"
      fi
    done
    APP_ROUTES="${routes}"
  fi

  load_app_env "${name}"
}

app_env_file() {
  echo "${ENV_DIR}/$1.env"
}

# Generated compose override that passes the app's variables into every service.
app_env_override() {
  echo "${STACKS_DIR}/$1/.panel-env.yaml"
}

# Generated compose override that publishes the ports routes point at.
app_ports_override() {
  echo "${STACKS_DIR}/$1/.panel-ports.yaml"
}

# The compose project the app's containers belong to: the app's name, unless
# the manifest keeps an older one (see adopt_legacy_project).
compose_project() {
  echo "${APP_COMPOSE_PROJECT:-${CURRENT_APP}}"
}

# The app's compose file followed by whichever generated overrides exist.
refresh_compose_args() {
  local override
  COMPOSE_ARGS=(-p "$(compose_project)" -f "${APP_COMPOSE_FILE}")
  # Other git apps may share a kept legacy project; their containers would
  # look like orphans.
  ORPHAN_ARGS=(--remove-orphans)
  [[ "$(compose_project)" == "${CURRENT_APP}" ]] || ORPHAN_ARGS=()
  for override in "$(app_ports_override "${CURRENT_APP}")" "$(app_env_override "${CURRENT_APP}")"; do
    if [[ -f "${override}" ]]; then
      COMPOSE_ARGS+=(-f "${override}")
    fi
  done
}

load_app_env() {
  local name="$1"
  local file line key override
  file="$(app_env_file "${name}")"
  APP_ENV_ARGS=()
  APP_ENV_KEYS=()
  if [[ -f "${file}" ]]; then
    while IFS= read -r line || [[ -n "${line}" ]]; do
      [[ -z "${line}" || "${line}" == \#* || "${line}" != *=* ]] && continue
      key="${line%%=*}"
      [[ "${key}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
      APP_ENV_ARGS+=("${line}")
      APP_ENV_KEYS+=("${key}")
    done <"${file}"
  fi

  refresh_compose_args
}

# (Re)generate the env override before containers are created. Variables are
# always available for ${VAR} interpolation; with APP_ENV_INJECT (default true)
# every service also receives them. Bare keys make compose read the values from
# its own environment, so values never have to be quoted into YAML.
prepare_env_override() {
  local name="$1"
  local override services svc key tmp
  override="$(app_env_override "${name}")"

  if [[ ${#APP_ENV_KEYS[@]} -eq 0 || "${APP_ENV_INJECT:-true}" != "true" ]]; then
    rm -f "${override}"
    refresh_compose_args
    return
  fi

  services="$(run_compose -f "${APP_COMPOSE_FILE}" config --services 2>/dev/null)" \
    || fail "could not list compose services to pass environment variables (is the compose file valid?)"

  tmp="$(mktemp)"
  {
    echo "# Generated by panelctl from the app's environment variables. Do not edit."
    echo "services:"
    while IFS= read -r svc; do
      [[ "${svc}" =~ ^[A-Za-z0-9._-]+$ ]] || continue
      printf '  "%s":\n    environment:\n' "${svc}"
      for key in "${APP_ENV_KEYS[@]}"; do
        printf '      - %s\n' "${key}"
      done
    done <<<"${services}"
  } >"${tmp}"
  install -m 0640 "${tmp}" "${override}"
  rm -f "${tmp}"
  refresh_compose_args
}

# (Re)generate the ports override from routes that target a service. Each one
# publishes the service's container port on the 127.0.0.1 port Caddy proxies
# to, so compose files don't need a ports: section for the panel's routes.
# A route with an empty service name uses the compose file's only service.
prepare_ports_override() {
  local name="$1"
  local override services="" entry domain upstream path target svc cport hport tmp
  local -a lines=() seen=()
  override="$(app_ports_override "${name}")"

  IFS=',' read -ra route_entries <<< "${APP_ROUTES}"
  for entry in "${route_entries[@]}"; do
    entry="$(echo "${entry}" | xargs)"
    IFS='|' read -r domain upstream path target <<< "${entry}"
    [[ -n "${target}" ]] || continue
    if [[ -z "${services}" ]]; then
      services="$(run_compose -f "${APP_COMPOSE_FILE}" config --services 2>/dev/null)" \
        || fail "could not list compose services to publish route ports (is the compose file valid?)"
    fi
    svc="${target%:*}"
    cport="${target##*:}"
    hport="${upstream##*:}"
    if [[ -z "${svc}" ]]; then
      [[ "$(grep -c . <<<"${services}")" -eq 1 ]] \
        || fail "route ${domain} targets port ${cport} but the compose file has several services; pick one in the app's domains"
      svc="${services}"
    fi
    grep -qxF -- "${svc}" <<<"${services}" \
      || fail "route ${domain} targets service '${svc}', which isn't in the compose file"
    [[ " ${seen[*]:-} " == *" ${svc}:${cport}:${hport} "* ]] && continue
    seen+=("${svc}:${cport}:${hport}")
    lines+=("${svc}|127.0.0.1:${hport}:${cport}")
  done

  if [[ ${#lines[@]} -eq 0 ]]; then
    rm -f "${override}"
    refresh_compose_args
    return
  fi

  tmp="$(mktemp)"
  {
    echo "# Generated by panelctl from the app's routes. Do not edit."
    echo "services:"
    local current="" line
    while IFS= read -r line; do
      svc="${line%%|*}"
      if [[ "${svc}" != "${current}" ]]; then
        printf '  "%s":\n    ports:\n' "${svc}"
        current="${svc}"
      fi
      printf '      - "%s"\n' "${line#*|}"
    done < <(printf '%s\n' "${lines[@]}" | sort)
  } >"${tmp}"
  install -m 0640 "${tmp}" "${override}"
  rm -f "${tmp}"
  refresh_compose_args
}

podman_command() {
  if command -v podman >/dev/null 2>&1; then
    command -v podman
  elif [[ -x /run/current-system/sw/bin/podman ]]; then
    echo /run/current-system/sw/bin/podman
  else
    fail "podman is not installed"
  fi
}

run_podman() {
  local podman_bin
  podman_bin="$(podman_command)"
  ensure_podman_runtime_env
  "${podman_bin}" "$@"
}

compose_command() {
  local podman_bin=""
  local podman_compose_bin=""

  if command -v podman >/dev/null 2>&1; then
    podman_bin="$(command -v podman)"
  elif [[ -x /run/current-system/sw/bin/podman ]]; then
    podman_bin="/run/current-system/sw/bin/podman"
  fi

  if command -v podman-compose >/dev/null 2>&1; then
    podman_compose_bin="$(command -v podman-compose)"
  elif [[ -x /run/current-system/sw/bin/podman-compose ]]; then
    podman_compose_bin="/run/current-system/sw/bin/podman-compose"
  fi

  if [[ -n "${podman_bin}" ]] && "${podman_bin}" compose version >/dev/null 2>&1; then
    echo "${podman_bin} compose"
    return
  fi

  if [[ -n "${podman_compose_bin}" ]]; then
    echo "${podman_compose_bin}"
    return
  fi

  fail "no compose command available (need 'podman compose' or 'podman-compose')"
}

ensure_podman_runtime_env() {
  local uid
  uid="$(id -u)"

  if [[ -z "${HOME:-}" ]]; then
    HOME="$(getent passwd "${uid}" | cut -d: -f6 || true)"
    export HOME
  fi

  if [[ -z "${XDG_RUNTIME_DIR:-}" ]]; then
    XDG_RUNTIME_DIR="/run/user/${uid}"
    export XDG_RUNTIME_DIR
  fi

  if [[ ! -d "${XDG_RUNTIME_DIR}" ]]; then
    fail "XDG_RUNTIME_DIR '${XDG_RUNTIME_DIR}' does not exist for uid ${uid}. Ensure user runtime is available (e.g. loginctl enable-linger $(id -un))."
  fi

  if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" && -S "${XDG_RUNTIME_DIR}/bus" ]]; then
    DBUS_SESSION_BUS_ADDRESS="unix:path=${XDG_RUNTIME_DIR}/bus"
    export DBUS_SESSION_BUS_ADDRESS
  fi

  unset DOCKER_HOST
  unset CONTAINER_HOST
}

run_compose() {
  local compose
  compose="$(compose_command)"

  ensure_podman_runtime_env

  if [[ "${compose}" == *" compose" ]]; then
    local podman_bin="${compose% compose}"
    env "${APP_ENV_ARGS[@]}" "${podman_bin}" compose "$@"
    return
  fi

  env "${APP_ENV_ARGS[@]}" "${compose}" "$@"
}

write_default_compose() {
  local name="$1"
  local routes="$2"
  local stack_dir
  local volume_dir
  stack_dir="$(app_stack_dir "${name}")"
  volume_dir="$(app_volume_dir "${name}")"

  local first_route="${routes%%,*}"
  local domain upstream path target
  IFS='|' read -r domain upstream path target <<< "${first_route}"

  if [[ -n "${target}" ]]; then
    # The route publishes the port (see prepare_ports_override).
    cat >"${stack_dir}/compose.yaml" <<EOF
services:
  app:
    image: docker.io/traefik/whoami:latest
    restart: unless-stopped
    volumes:
      - ${volume_dir}/data:/data
EOF
    return
  fi

  # Older style: publish the first route's upstream port directly.
  cat >"${stack_dir}/compose.yaml" <<EOF
services:
  app:
    image: docker.io/traefik/whoami:latest
    restart: unless-stopped
    ports:
      - "127.0.0.1:${upstream##*:}:80"
    volumes:
      - ${volume_dir}/data:/data
EOF
}

write_manifest() {
  local name="$1"
  local routes="$2"
  local auth="$3"
  local manifest
  local stack_dir
  local volume_dir
  local route_file

  manifest="$(app_manifest "${name}")"
  stack_dir="$(app_stack_dir "${name}")"
  volume_dir="$(app_volume_dir "${name}")"
  route_file="$(app_route_file)"

  cat >"${manifest}" <<EOF
APP_NAME="${name}"
APP_ROUTES="${routes}"
APP_AUTH_PROTECTED="${auth}"
APP_STACK_DIR="${stack_dir}"
APP_COMPOSE_FILE="${stack_dir}/compose.yaml"
APP_VOLUME_DIR="${volume_dir}"
APP_ROUTE_FILE="${route_file}"
EOF
}

cmd_init() {
  local name="$1"
  local routes="$2"
  local auth="${3:-true}"

  validate_name "${name}"
  validate_routes "${routes}"
  [[ "${auth}" == "true" || "${auth}" == "false" ]] || fail "auth must be true or false"

  ensure_base_dirs

  local manifest
  local stack_dir
  local volume_dir
  manifest="$(app_manifest "${name}")"
  stack_dir="$(app_stack_dir "${name}")"
  volume_dir="$(app_volume_dir "${name}")"

  [[ ! -f "${manifest}" ]] || fail "app '${name}' already exists"

  mkdir -p "${stack_dir}" "${volume_dir}/data"
  write_default_compose "${name}" "${routes}"
  write_manifest "${name}" "${routes}" "${auth}"
  cmd_render_route "${name}"

  log info "initialized app '${name}'"
}

cmd_render_route() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  local auth_block=""
  if [[ "${APP_AUTH_PROTECTED}" == "true" ]]; then
    auth_block="${FORWARD_AUTH_BLOCK}"
  fi

  local route_file
  route_file="$(app_route_file)"

  routes_lock

  # Strip any existing block for this app from the aggregate file.
  local tmp
  tmp="$(mktemp)"

  if [[ -f "${route_file}" ]]; then
    sed "/^# route:${name}:start$/,/^# route:${name}:end$/d" "${route_file}" >"${tmp}" || true
  else
    printf "" >"${tmp}"
  fi

  {
    printf "# route:%s:start\n" "${name}"
    IFS=',' read -ra route_entries <<< "${APP_ROUTES}"
    for entry in "${route_entries[@]}"; do
      entry="$(echo "${entry}" | xargs)"
      IFS='|' read -r domain upstream path target <<< "${entry}"
      # If upstream is empty (no second pipe), this is the old format
      if [[ -z "${upstream}" ]]; then
        upstream="${path}"
        path=""
      fi
      if [[ -n "${path}" ]]; then
        if [[ -n "${auth_block}" ]]; then
          printf "%s {\n%s  reverse_proxy %s %s\n}\n" "${domain}" "${auth_block}" "${path}" "${upstream}"
        else
          printf "%s {\n  reverse_proxy %s %s\n}\n" "${domain}" "${path}" "${upstream}"
        fi
      else
        if [[ -n "${auth_block}" ]]; then
          printf "%s {\n%s  reverse_proxy %s\n}\n" "${domain}" "${auth_block}" "${upstream}"
        else
          printf "%s {\n  reverse_proxy %s\n}\n" "${domain}" "${upstream}"
        fi
      fi
    done
    printf "# route:%s:end\n" "${name}"
  } >>"${tmp}"

  install -m 0664 -o "${PANEL_USER}" -g "${PANEL_GROUP}" "${tmp}" "${route_file}"
  rm -f "${tmp}"
  routes_unlock
  log info "rendered route ${route_file}"
}

# Before panelctl passed -p, compose named the project after the compose
# file's directory, so every git app (repo/compose.yaml) shared the project
# "repo". Move the app's old containers out of the way so they are recreated
# under its own name. Named volumes are prefixed with the project name, so an
# app whose containers use them keeps its old project instead.
adopt_legacy_project() {
  [[ -z "${APP_COMPOSE_PROJECT:-}" ]] || return 0
  local dir id project legacy="" volumes
  local ids=()
  dir="$(cd "$(dirname "${APP_COMPOSE_FILE}")" 2>/dev/null && pwd -P)" || return 0

  while IFS='|' read -r id project; do
    [[ -n "${id}" && -n "${project}" && "${project}" != "${CURRENT_APP}" ]] || continue
    ids+=("${id}")
    legacy="${project}"
  done < <(run_podman ps --all --filter "label=com.docker.compose.project.working_dir=${dir}" \
    --format '{{.ID}}|{{index .Labels "com.docker.compose.project"}}' 2>/dev/null || true)
  [[ ${#ids[@]} -gt 0 ]] || return 0

  volumes="$(run_podman inspect --format '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' "${ids[@]}" 2>/dev/null | xargs || true)"
  if [[ -n "${volumes}" ]]; then
    echo "APP_COMPOSE_PROJECT=\"${legacy}\"" >>"$(app_manifest "${CURRENT_APP}")"
    APP_COMPOSE_PROJECT="${legacy}"
    refresh_compose_args
    echo "Keeping compose project '${legacy}', because the app's containers use named volumes (${volumes})"
    log warning "app '${CURRENT_APP}' keeps compose project '${legacy}' because its containers use named volumes"
    return 0
  fi

  echo "Moving the app's containers from compose project '${legacy}' to '${CURRENT_APP}'"
  run_podman rm --force "${ids[@]}" >/dev/null || fail "could not remove the old containers"
  # The shared pod goes away once no other app's containers are left in it.
  run_podman pod rm "pod_${legacy}" >/dev/null 2>&1 || true
}

# True when a container of the app runs an older image than its tag now
# points at, e.g. after a rebuild. podman-compose only recreates containers
# when the compose file itself changes, so without this a push that only
# changes code would build a new image and keep running the old one.
containers_outdated() {
  local id image current
  while IFS='|' read -r id image; do
    [[ -n "${id}" && -n "${image}" ]] || continue
    current="$(run_podman image inspect --format '{{.Id}}' "${image}" 2>/dev/null)" || continue
    [[ "${current}" == "${id}"* || "${id}" == "${current}"* ]] || return 0
  done < <(run_podman ps --all --filter "label=com.docker.compose.project=$(compose_project)" \
    --format '{{.ImageID}}|{{.Image}}' 2>/dev/null || true)
  return 1
}

cmd_deploy() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  log info "Starting deployment for app '${name}'"

  cmd_render_route "${name}"
  prepare_ports_override "${name}"
  prepare_env_override "${name}"
  adopt_legacy_project

  # Stream compose output as it happens (the panel shows it live in the
  # deployment log) and keep a copy in the journal.
  if ! run_compose "${COMPOSE_ARGS[@]}" build 2>&1 | tee >(journal_copy); then
    log err "Deployment failed for app '${name}'"
    fail "image build failed"
  fi

  local up_args=(up -d --no-build "${ORPHAN_ARGS[@]}")
  if containers_outdated; then
    echo "Images changed, recreating the app's containers"
    up_args+=(--force-recreate)
  fi
  if ! run_compose "${COMPOSE_ARGS[@]}" "${up_args[@]}" 2>&1 | tee >(journal_copy); then
    log err "Deployment failed for app '${name}'"
    fail "compose up failed"
  fi

  log info "Successfully deployed app '${name}'"
}

cmd_restart() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  log info "Restarting app '${name}'"

  adopt_legacy_project
  run_compose "${COMPOSE_ARGS[@]}" down "${ORPHAN_ARGS[@]}" || fail "compose down failed"
  prepare_ports_override "${name}"
  prepare_env_override "${name}"

  if ! run_compose "${COMPOSE_ARGS[@]}" up -d --build "${ORPHAN_ARGS[@]}" 2>&1; then
    fail "compose up failed during restart"
  fi

  log info "restarted app '${name}'"
}

cmd_stop() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  run_compose "${COMPOSE_ARGS[@]}" down "${ORPHAN_ARGS[@]}" || fail "compose down failed"
  log info "stopped app '${name}'"
}

cmd_status() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  run_compose "${COMPOSE_ARGS[@]}" ps --format json 2>/dev/null || \
    run_compose "${COMPOSE_ARGS[@]}" ps 2>/dev/null || \
    log info "no containers running"
}

cmd_logs() {
  local name="$1"
  shift
  validate_name "${name}"
  load_app "${name}"

  local tail_lines="100"
  local -a extra=() services=()
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --tail)
        [[ "${2:-}" =~ ^[0-9]+$ ]] || fail "--tail needs a number"
        tail_lines="$2"
        shift 2
        ;;
      --follow|-f)
        extra+=(--follow)
        shift
        ;;
      --service)
        [[ "${2:-}" =~ ^[A-Za-z0-9._-]+$ ]] || fail "invalid service name"
        services+=("$2")
        shift 2
        ;;
      *)
        shift
        ;;
    esac
  done

  run_compose "${COMPOSE_ARGS[@]}" logs --tail "${tail_lines}" "${extra[@]}" "${services[@]}" 2>&1 || log info "no logs available"
}

cmd_services() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"
  run_compose -f "${APP_COMPOSE_FILE}" config --services
}

# Every container of the user, with labels (the panel maps them to apps).
cmd_containers() {
  run_podman ps --all --format json
}

# One sample of resource usage for all running containers.
cmd_stats() {
  run_podman stats --no-stream --format json
}

# Interactive shell in one of the app's containers (used by the web terminal).
cmd_exec() {
  local name="$1"
  local container="$2"
  validate_name "${name}"
  load_app "${name}"
  [[ "${container}" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] || fail "invalid container name"

  local labels
  labels="$(run_podman inspect --format '{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "io.podman.compose.project"}}' "${container}" 2>/dev/null)" \
    || fail "container '${container}' does not exist"
  [[ "|${labels}|" == *"|${name}|"* ]] || fail "container '${container}' does not belong to app '${name}'"

  exec "$(podman_command)" exec -it -e TERM="${TERM:-xterm-256color}" "${container}" \
    sh -c 'if command -v bash >/dev/null 2>&1; then exec bash -l; else exec sh -l; fi'
}

cmd_validate_compose() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  if run_compose "${COMPOSE_ARGS[@]}" config >/dev/null 2>&1; then
    log info "compose file is valid"
  else
    local output
    output="$(run_compose "${COMPOSE_ARGS[@]}" config 2>&1 || true)"
    fail "compose validation failed: ${output}"
  fi
}

cmd_remove() {
  local name="$1"
  local keep_volumes="${2:-}"
  validate_name "${name}"
  load_app "${name}"

  run_compose "${COMPOSE_ARGS[@]}" down "${ORPHAN_ARGS[@]}" 2>/dev/null || true

  # Remove this app's block from the aggregate routes file.
  local route_file
  route_file="$(app_route_file)"
  if [[ -f "${route_file}" ]]; then
    routes_lock
    local tmp
    tmp="$(mktemp)"
    sed "/^# route:${name}:start$/,/^# route:${name}:end$/d" "${route_file}" >"${tmp}" || true
    install -m 0664 -o "${PANEL_USER}" -g "${PANEL_GROUP}" "${tmp}" "${route_file}"
    rm -f "${tmp}"
    routes_unlock
  fi

  rm -f "$(app_manifest "${name}")" "$(app_env_file "${name}")"
  rm -rf "${APP_STACK_DIR}"

  if [[ "${keep_volumes}" != "--keep-volumes" ]]; then
    rm -rf "${APP_VOLUME_DIR}"
  fi

  log info "removed app '${name}'"
}

cmd_set_routes() {
  local name="$1"
  local routes="$2"
  local manifest

  validate_name "${name}"
  validate_routes "${routes}"
  manifest="$(app_manifest "${name}")"
  [[ -f "${manifest}" ]] || fail "app '${name}' does not exist"

  # Update APP_ROUTES in the manifest file, strip old fields, preserve others
  local tmp
  tmp="$(mktemp)"
  local found_routes=false
  while IFS= read -r line; do
    case "${line}" in
      APP_ROUTES=*)
        printf 'APP_ROUTES="%s"\n' "${routes}" >> "${tmp}"
        found_routes=true
        ;;
      APP_DOMAIN=*|APP_DOMAINS=*|APP_PORT=*|APP_UPSTREAM=*)
        # Strip old format fields
        ;;
      *)
        printf '%s\n' "${line}" >> "${tmp}"
        ;;
    esac
  done < "${manifest}"
  if ! "${found_routes}"; then
    printf 'APP_ROUTES="%s"\n' "${routes}" >> "${tmp}"
  fi
  install -m 0664 "${tmp}" "${manifest}"

  # Re-render Caddy routes (auto-reloads via systemd.path watcher)
  cmd_render_route "${name}"

  log info "updated routes for app '${name}'. Edit compose file if new ports need exposing."
}

cmd_backup() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  ensure_base_dirs

  local volume_dir
  volume_dir="$(app_volume_dir "${name}")"
  [[ -d "${volume_dir}" ]] || fail "volume directory '${volume_dir}' does not exist"

  local timestamp
  timestamp="$(date +%Y%m%d-%H%M%S)"
  local backup_file="${BACKUPS_DIR}/${name}-${timestamp}.zip"

  # Stop containers before backup for consistency
  local was_running=false
  if run_compose "${COMPOSE_ARGS[@]}" ps --format json 2>/dev/null | grep -q '"running"' 2>/dev/null; then
    was_running=true
    log info "stopping containers for consistent backup..."
    run_compose "${COMPOSE_ARGS[@]}" down 2>/dev/null || true
  fi

  (cd "${volume_dir}" && zip -r "${backup_file}" .) || fail "zip failed"

  # Also include the compose file in the backup
  local stack_dir
  stack_dir="$(app_stack_dir "${name}")"
  if [[ -f "${stack_dir}/compose.yaml" ]]; then
    (cd "${stack_dir}" && zip -j "${backup_file}" compose.yaml) || true
  fi

  # Restart if it was running
  if [[ "${was_running}" == "true" ]]; then
    log info "restarting containers after backup..."
    run_compose "${COMPOSE_ARGS[@]}" up -d 2>/dev/null || true
  fi

  local size
  size="$(du -h "${backup_file}" | cut -f1)"
  log info "backup created: ${backup_file} (${size})"
}

cmd_list_backups() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  ensure_base_dirs

  local found=0
  for bf in "${BACKUPS_DIR}/${name}"-*.zip; do
    [[ -e "${bf}" ]] || continue
    found=1
    local fname size mtime
    fname="$(basename "${bf}")"
    size="$(du -h "${bf}" | cut -f1)"
    mtime="$(stat -c '%Y' "${bf}" 2>/dev/null || stat -f '%m' "${bf}" 2>/dev/null || echo "0")"
    echo "${fname} ${size} ${mtime}"
  done

  if [[ "${found}" -eq 0 ]]; then
    log info "no backups found for '${name}'"
  fi
}

cmd_volume_clear() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  log info "clearing volume data for app '${name}'"
  run_compose "${COMPOSE_ARGS[@]}" down "${ORPHAN_ARGS[@]}" 2>/dev/null || true

  local data_dir="${APP_VOLUME_DIR}/data"
  if [[ -d "${data_dir}" ]]; then
    rm -rf "${data_dir:?}"/*
    rm -rf "${data_dir:?}"/.[!.]* 2>/dev/null || true
  fi
  mkdir -p "${APP_VOLUME_DIR}/data"

  log info "volume data cleared for app '${name}'"
}

cmd_restore() {
  local name="$1"
  local backup_file="$2"
  validate_name "${name}"
  load_app "${name}"

  # Resolve backup file path
  local full_path="${backup_file}"
  if [[ ! -f "${full_path}" ]]; then
    full_path="${BACKUPS_DIR}/${backup_file}"
  fi
  [[ -f "${full_path}" ]] || fail "backup file '${backup_file}' not found"

  # Ensure it's a zip file within the backups directory
  local norm_path
  norm_path="$(realpath "${full_path}")"
  local norm_backups
  norm_backups="$(realpath "${BACKUPS_DIR}")"
  [[ "${norm_path}" == "${norm_backups}"/* ]] || fail "backup file must be in the backups directory"

  local volume_dir
  volume_dir="$(app_volume_dir "${name}")"

  # Stop containers before restore
  log info "stopping containers for restore..."
  run_compose "${COMPOSE_ARGS[@]}" down 2>/dev/null || true

  # Clear existing volume data and extract backup
  rm -rf "${volume_dir:?}"/*
  mkdir -p "${volume_dir}"
  (cd "${volume_dir}" && unzip -o "${norm_path}") || fail "unzip failed"

  log info "restored '${name}' from $(basename "${norm_path}")"
  log info "run 'panelctl deploy ${name}' to start the app'"
}

cmd_inspect_volumes() {
  local name="$1"
  validate_name "${name}"
  load_app "${name}"

  echo "default|${APP_VOLUME_DIR}/data"

  {
    run_podman volume ls --filter label=com.docker.compose.project="${name}" --format '{{.Name}}|{{.Mountpoint}}' 2>/dev/null || true
    run_podman volume ls --filter label=io.podman.compose.project="${name}" --format '{{.Name}}|{{.Mountpoint}}' 2>/dev/null || true
  # grep exits 1 when there are no named volumes; that is not an error.
  } | sort -u | grep -v '^$' || true
}

cmd_list() {
  ensure_base_dirs
  local found=0
  for mf in "${APPS_DIR}"/*.env; do
    [[ -e "${mf}" ]] || continue
    found=1
    # shellcheck disable=SC1090
    source /dev/null  # reset any leftover variables
    unset APP_REPO_URL APP_REPO_BRANCH APP_REPO_DIR 2>/dev/null || true
    source "${mf}"
    # Backward compat: build APP_ROUTES from old format
    local routes="${APP_ROUTES:-}"
    if [[ -z "${routes}" && -n "${APP_DOMAIN:-}" ]]; then
      local upstream="${APP_UPSTREAM:-127.0.0.1:${APP_PORT:-18080}}"
      local domains_str="${APP_DOMAINS:-${APP_DOMAIN}}"
      IFS=',' read -ra domain_arr <<< "${domains_str}"
      for d in "${domain_arr[@]}"; do
        d="$(echo "${d}" | xargs)"
        if [[ -n "${routes}" ]]; then
          routes="${routes},${d}|${upstream}"
        else
          routes="${d}|${upstream}"
        fi
      done
    fi
    # Show abbreviated: first route's domain + upstream, and count
    local first_route="${routes%%,*}"
    local route_count=1
    if [[ "${routes}" == *","* ]]; then
      route_count="$(( $(grep -o ',' <<< "${routes}" | wc -l) + 1 ))"
    fi
    local repo_info="${APP_REPO_URL:-}"
    echo "${APP_NAME} ${first_route} routes=${route_count} auth=${APP_AUTH_PROTECTED} ${repo_info}"
  done

  if [[ "${found}" -eq 0 ]]; then
    log info "no apps found"
  fi
}

cmd_show() {
  local name="$1"
  validate_name "${name}"
  local mf
  mf="$(app_manifest "${name}")"
  [[ -f "${mf}" ]] || fail "app '${name}' does not exist"
  cat "${mf}"
}

main() {
  local cmd="${1:-}"

  case "${cmd}" in
    init)
      [[ $# -ge 3 ]] || fail "usage: panelctl init <name> <routes> [auth]"
      cmd_init "$2" "$3" "${4:-true}"
      ;;
    set-routes)
      [[ $# -eq 3 ]] || fail "usage: panelctl set-routes <name> <routes>"
      cmd_set_routes "$2" "$3"
      ;;
    render-route)
      [[ $# -eq 2 ]] || fail "usage: panelctl render-route <name>"
      cmd_render_route "$2"
      ;;
    deploy)
      [[ $# -eq 2 ]] || fail "usage: panelctl deploy <name>"
      cmd_deploy "$2"
      ;;
    restart)
      [[ $# -eq 2 ]] || fail "usage: panelctl restart <name>"
      cmd_restart "$2"
      ;;
    stop)
      [[ $# -eq 2 ]] || fail "usage: panelctl stop <name>"
      cmd_stop "$2"
      ;;
    status)
      [[ $# -eq 2 ]] || fail "usage: panelctl status <name>"
      cmd_status "$2"
      ;;
    logs)
      [[ $# -ge 2 ]] || fail "usage: panelctl logs <name> [--tail N] [--follow] [--service S]"
      cmd_logs "$2" "${@:3}"
      ;;
    services)
      [[ $# -eq 2 ]] || fail "usage: panelctl services <name>"
      cmd_services "$2"
      ;;
    containers)
      [[ $# -eq 1 ]] || fail "usage: panelctl containers"
      cmd_containers
      ;;
    stats)
      [[ $# -eq 1 ]] || fail "usage: panelctl stats"
      cmd_stats
      ;;
    exec)
      [[ $# -eq 3 ]] || fail "usage: panelctl exec <name> <container>"
      cmd_exec "$2" "$3"
      ;;
    validate-compose)
      [[ $# -eq 2 ]] || fail "usage: panelctl validate-compose <name>"
      cmd_validate_compose "$2"
      ;;
    remove)
      [[ $# -ge 2 ]] || fail "usage: panelctl remove <name> [--keep-volumes]"
      cmd_remove "$2" "${3:-}"
      ;;
    backup)
      [[ $# -eq 2 ]] || fail "usage: panelctl backup <name>"
      cmd_backup "$2"
      ;;
    list-backups)
      [[ $# -eq 2 ]] || fail "usage: panelctl list-backups <name>"
      cmd_list_backups "$2"
      ;;
    restore)
      [[ $# -eq 3 ]] || fail "usage: panelctl restore <name> <backup-file>"
      cmd_restore "$2" "$3"
      ;;
    volume-clear)
      [[ $# -eq 2 ]] || fail "usage: panelctl volume-clear <name>"
      cmd_volume_clear "$2"
      ;;
    inspect-volumes)
      [[ $# -eq 2 ]] || fail "usage: panelctl inspect-volumes <name>"
      cmd_inspect_volumes "$2"
      ;;
    list)
      [[ $# -eq 1 ]] || fail "usage: panelctl list"
      cmd_list
      ;;
    show)
      [[ $# -eq 2 ]] || fail "usage: panelctl show <name>"
      cmd_show "$2"
      ;;
    ""|-h|--help|help)
      usage
      ;;
    *)
      fail "unknown command '${cmd}'"
      ;;
  esac
}

main "$@"