Merge pull request 'panel: redesign web UI and make syncing responsive' (#1) from agent/panel-ui-improve into main

Reviewed-on: https://git.srazka.com/reudy-net/nixos/pulls/1

panel.nix

@@ -1,5 +1,8 @@
-{ config, pkgs, ... }:
+{ config, lib, pkgs, ... }:
 
+let
+  forgejoServer = config.services.forgejo.settings.server;
+in
 {
   environment.systemPackages = [
     (pkgs.writeShellScriptBin "panelctl" (builtins.readFile ./panel/panelctl.sh))
@@ -33,6 +36,8 @@
       pkgs.zip
       pkgs.unzip
       pkgs.git
+      pkgs.util-linux # flock, used by panelctl to serialise routes file writes
+      pkgs.openssh # cloning repositories over ssh with the panel's deploy key
     ];
 
     serviceConfig = {
@@ -51,6 +56,12 @@
       PANEL_BASE_DIR = "/var/lib/containers";
       PANELCTL_PATH = "/run/current-system/sw/bin/panelctl";
       PANEL_FRONTEND_DIR = "${./panel/frontend}";
+
+      # Forgejo integration (repo picker, private clones, commit links).
+      # The API is reached on localhost; clones use the public URLs.
+      PANEL_FORGEJO_URL = lib.removeSuffix "/" forgejoServer.ROOT_URL;
+      PANEL_FORGEJO_API_URL = "http://${forgejoServer.HTTP_ADDR}:${toString forgejoServer.HTTP_PORT}";
+      PANEL_FORGEJO_SSH_URL = "ssh://${forgejoServer.BUILTIN_SSH_SERVER_USER}@${forgejoServer.DOMAIN}:${toString forgejoServer.SSH_PORT}";
     };
   };
 

panel/API.md

@@ -12,6 +12,11 @@ Default bind: `127.0.0.1:9911`
 |--------|------|-------------|
 | GET | `/` | Web UI (served from `frontend/index.html`) |
 | GET | `/health` | Health check |
+| GET | `/status` | All apps with routes, container status and running operation (what the UI polls) |
+| GET | `/integrations` | Forgejo connection (`configured`, `url`, `has_token`, `user`) and the SSH deploy public key |
+| POST | `/integrations/forgejo` | `{"token": "..."}` — verify against Forgejo and store; `""` disconnects |
+| GET | `/forgejo/repos?q=` | Search repositories visible to the stored token (public ones without) |
+| GET | `/forgejo/branches?repo=owner/name` | Branch names of a Forgejo repository |
 
 ### Apps — Read
 
@@ -25,6 +30,14 @@ Default bind: `127.0.0.1:9911`
 | GET | `/apps/<name>/logs?tail=N` | Fetch last N log lines (default 100) |
 | GET | `/apps/<name>/backups` | List available backups |
 | GET | `/apps/<name>/backups/<file>` | Download backup zip |
+| GET | `/apps/<name>/env` | Environment variables: `{"vars": [{"key", "value"}], "inject": true}` |
+| GET | `/apps/<name>/repo` | Git source info (URL, web URL, provider, branch, deployed commit, local changes, deploy key for ssh) |
+| GET | `/apps/<name>/repo?fetch=1` | Same, plus fetches the remote and reports `behind` / `remote` |
+| GET | `/apps/<name>/volumes` | Volumes the file browser can open |
+| GET | `/apps/<name>/volume/files?vol=&path=` | List a folder in a volume |
+| GET | `/apps/<name>/volume/download?vol=&path=` | Download a file from a volume |
+| PUT | `/apps/<name>/volume/files?vol=&path=` | Upload a file (raw body) |
+| DELETE | `/apps/<name>/volume/files?vol=&path=` | Delete a file or folder |
 
 ### Apps — Write
 
@@ -41,6 +54,79 @@ Default bind: `127.0.0.1:9911`
 | POST | `/apps/<name>/backup` | Create volume backup (zip) |
 | POST | `/apps/<name>/restore` | Restore from backup |
 | POST | `/apps/<name>/remove` | Remove app |
+| POST | `/apps/<name>/repo-pull` | Git apps: fetch branch, hard-reset checkout to it, redeploy |
+| POST | `/apps/<name>/env` | Replace environment variables: `{"vars": [...], "inject": true, "deploy": false}` |
+| POST | `/apps/<name>/volume-clear` | Stop the app and empty its default data folder |
+
+Write operations are serialised per app. While one runs, another write to the
+same app returns `409` with `{"ok": false, "error": "...", "busy": "deploy"}`.
+`deploy` returns the compose output in `stdout` (or `stderr` on failure).
+
+### Create app (git repository, with environment variables)
+
+```json
+{
+  "name": "blog",
+  "routes": [{"domain": "blog.srazka.com", "upstream": "127.0.0.1:18090"}],
+  "auth": true,
+  "source_type": "git",
+  "repo_url": "https://git.srazka.com/reudy-net/blog.git",
+  "repo_branch": "",
+  "use_forgejo_token": true,
+  "env": [{"key": "DATABASE_URL", "value": "postgres://..."}],
+  "env_inject": true
+}
+```
+
+- `repo_url` may be `https://…`, `ssh://git@host:port/owner/repo.git` or
+  `git@host:owner/repo.git`. ssh URLs use the panel's deploy key.
+- `repo_token` sets an https token explicitly; `use_forgejo_token` uses the
+  token stored in Settings (only for URLs on the configured Forgejo host).
+- An empty branch uses the repository's default branch. The compose file must
+  be at the repository root.
+- The older `source_type: "github"` with `github_url` / `github_branch` /
+  `github_pat` is still accepted.
+
+### Sync response (`repo-pull`)
+
+```json
+{
+  "ok": true,
+  "stdout": "HEAD is now at d7df557 Bump image tag\n...compose output...",
+  "before": {"sha": "4fb7976...", "short": "4fb7976", "subject": "Initial compose", "author": "reudy", "time": 1790460618},
+  "after": {"sha": "d7df557...", "short": "d7df557", "subject": "Bump image tag", "author": "reudy", "time": 1790460643},
+  "changed": true
+}
+```
+
+### Status response (`/status`)
+
+```json
+{
+  "ok": true,
+  "time": 1790460650,
+  "apps": [
+    {
+      "name": "whoami",
+      "routes": [{"domain": "whoami.srazka.com", "upstream": "127.0.0.1:18080"}],
+      "auth": true,
+      "compose_file": "/var/lib/containers/stacks/whoami/compose.yaml",
+      "repo_url": "",
+      "repo_branch": "",
+      "busy": null,
+      "status": {
+        "state": "running",
+        "running": true,
+        "running_count": 1,
+        "total": 1,
+        "containers": [{"name": "whoami-app-1", "state": "running", "status": "Up 3 minutes", "image": "docker.io/traefik/whoami:latest", "running": true}]
+      }
+    }
+  ]
+}
+```
+
+`state` is one of `running`, `partial` (some containers down), `stopped` or `unknown`.
 
 ## Example payloads
 

panel/README.md

@@ -22,14 +22,19 @@ All app routes are written to a single `routes/routes.caddy` file that Caddy imp
 ## Quick workflow
 
 ```bash
-# Create a new app (single domain)
+# Routes are "domain|upstream[|path]" entries, comma-separated.
-panelctl init whoami whoami.srazka.com 18080 true
 
-# Create with multiple domains
+# Create a new app (single route, protected by Authelia)
-panelctl init myapp "app.srazka.com,www.srazka.com" 18081 true
+panelctl init whoami "whoami.srazka.com|127.0.0.1:18080" true
+
+# Create with multiple routes (different ports, optional path)
+panelctl init myapp "app.srazka.com|127.0.0.1:18081,api.srazka.com|127.0.0.1:18082|/api/*" true
 
 # Create with wildcard domain (requires DNS challenge in Caddy)
-panelctl init wild "*.srazka.com" 18082 false
+panelctl init wild "*.srazka.com|127.0.0.1:18083" false
+
+# Change routes later (Caddy reloads automatically)
+panelctl set-routes whoami "whoami.srazka.com|127.0.0.1:18080,who.srazka.com|127.0.0.1:18080"
 
 # Deploy (compose up + caddy reload)
 panelctl deploy whoami
@@ -91,9 +96,70 @@ panelctl remove whoami
 
 ### Web UI features
 
-- Create apps with multiple domains and wildcard support
+- Live status: one `/status` poll every few seconds (faster while something is
-- Live container status indicators (auto-refreshes)
+  running, paused when the tab is hidden) updates cards in place, so open tabs,
-- Deploy, restart, stop, remove from the UI
+  unsaved edits and scroll positions are never lost. The header shows when the
-- Inline compose editor with save, validate, and save+deploy
+  panel last synced and warns when the Authelia session has expired.
-- Log viewer with configurable tail length
+- Per-app status (running / partial / stopped), container list, and a busy
-- Volume backup management: create, list, download, restore
+  indicator that is shared between browsers while an operation runs.
+- New-app dialog: starter container, pasted compose file or git repository;
+  suggests the next free port and a domain based on the app name.
+- Compose editor with unsaved-changes tracking, Ctrl+S, save & deploy, validate.
+- Logs with follow mode, routes editor with validation, file browser with
+  drag-and-drop upload, backups with restore (and optional redeploy).
+- Git source tab: deployed commit, "check for updates", and sync & deploy.
+- Activity drawer with the output of every operation (e.g. why a deploy failed).
+- Keyboard: `/` search, `N` new app, `Esc` closes menus. Deep links like
+  `#/whoami/logs` open an app on a specific tab.
+
+### Git-backed apps
+
+Apps created from a repository (Forgejo, GitHub or any git host, over https or
+ssh) are cloned to `stacks/<app>/repo`.
+
+**Forgejo.** `panel.nix` points the panel at the local Forgejo
+(`PANEL_FORGEJO_URL`, `PANEL_FORGEJO_API_URL`, `PANEL_FORGEJO_SSH_URL`, taken
+from `forgejo.nix`). In the panel's **Settings** you can connect a Forgejo
+access token (read access to repositories and user). With it, the new-app
+dialog lists your repositories and branches, and private ones are cloned over
+https with the token. Without it, public repositories are listed and private
+ones are cloned over ssh with the deploy key. Commit and compare links point at
+Forgejo. The token is stored in `state/panel/forgejo-token` (mode 0600).
+
+**SSH / deploy key.** The panel generates an ed25519 key pair in
+`state/panel/ssh/` the first time it is needed. Its public half is shown in
+Settings (and next to ssh URLs); add it as a read-only deploy key to a
+repository — or to your Forgejo account for access to all repositories — to
+clone `ssh://git@git.srazka.com:14922/owner/repo.git` style URLs. **Sync** fetches the configured branch and
+hard-resets the checkout to it before redeploying, so the repository is the
+source of truth: compose edits made in the panel are discarded on the next sync
+(the UI warns about this). An access token for a private repository is stored in
+the clone's `.git/config`; use a read-only token.
+
+### Environment variables
+
+Each app can have environment variables (the **Environment** tab, or when
+creating the app; `.env` text can be pasted in). They are stored in
+`state/env/<app>.env` as `KEY=VALUE` lines (mode 0600) — outside the repository
+and stack directory, so git syncs never touch them — and `panelctl` passes them
+to every compose command:
+
+- They are always available for `${VAR}` interpolation in the compose file.
+  The UI points out variables the compose file uses without a default that
+  aren't set.
+- With **Pass to every container** (the default), `deploy`/`restart` also
+  generate `stacks/<app>/.panel-env.yaml`, a compose override that lists the
+  keys under every service's `environment:`. Compose reads the values from its
+  own environment, so they are never quoted into YAML, and they take
+  precedence over values set in the compose file.
+
+Values must be single-line. Names that would change how podman/compose run
+(`PATH`, `HOME`, `XDG_*`, `DOCKER_*`, `COMPOSE_*`, `PODMAN_*`, …) are rejected.
+Changes apply on the next deploy. Backups do not include variables.
+
+### Concurrency
+
+`panel-api` handles requests concurrently, so a long deploy never blocks status
+or logs. Mutating operations are serialised per app — a second operation on a
+busy app gets HTTP 409 — and `panelctl` takes a `flock` on the shared routes
+file while rewriting it.

panel/panelctl.sh

@@ -7,8 +7,15 @@ VOLUMES_DIR="${BASE_DIR}/volumes"
 ROUTES_DIR="${BASE_DIR}/routes"
 STATE_DIR="${BASE_DIR}/state"
 APPS_DIR="${STATE_DIR}/apps"
+ENV_DIR="${STATE_DIR}/env"
 BACKUPS_DIR="${BASE_DIR}/backups"
 
+# Set by load_app: compose file arguments, and the app's environment variables
+# as KEY=VALUE words (passed to compose via env(1), never sourced).
+COMPOSE_ARGS=()
+APP_ENV_ARGS=()
+APP_ENV_KEYS=()
+
 FORWARD_AUTH_BLOCK='  forward_auth 127.0.0.1:9091 {
     uri /api/authz/forward-auth
     copy_headers Remote-User Remote-Groups Remote-Email Remote-Name
@@ -149,6 +156,19 @@ app_route_file() {
   echo "${ROUTES_DIR}/routes.caddy"
 }
 
+# The routes file is shared by all apps and rewritten read-modify-write, so
+# concurrent panelctl runs (the API handles requests in parallel) must take turns.
+routes_lock() {
+  exec 9>"${ROUTES_DIR}/.routes.lock"
+  if command -v flock >/dev/null 2>&1; then
+    flock -w 30 9 || fail "timed out waiting for the routes file lock"
+  fi
+}
+
+routes_unlock() {
+  exec 9>&-
+}
+
 load_app() {
   local name="$1"
   local manifest
@@ -173,6 +193,75 @@ load_app() {
     done
     APP_ROUTES="${routes}"
   fi
+
+  load_app_env "${name}"
+}
+
+app_env_file() {
+  echo "${ENV_DIR}/$1.env"
+}
+
+# Generated compose override that passes the app's variables into every service.
+app_env_override() {
+  echo "${STACKS_DIR}/$1/.panel-env.yaml"
+}
+
+load_app_env() {
+  local name="$1"
+  local file line key override
+  file="$(app_env_file "${name}")"
+  APP_ENV_ARGS=()
+  APP_ENV_KEYS=()
+  if [[ -f "${file}" ]]; then
+    while IFS= read -r line || [[ -n "${line}" ]]; do
+      [[ -z "${line}" || "${line}" == \#* || "${line}" != *=* ]] && continue
+      key="${line%%=*}"
+      [[ "${key}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
+      APP_ENV_ARGS+=("${line}")
+      APP_ENV_KEYS+=("${key}")
+    done <"${file}"
+  fi
+
+  COMPOSE_ARGS=(-f "${APP_COMPOSE_FILE}")
+  override="$(app_env_override "${name}")"
+  if [[ -f "${override}" ]]; then
+    COMPOSE_ARGS+=(-f "${override}")
+  fi
+}
+
+# (Re)generate the env override before containers are created. Variables are
+# always available for ${VAR} interpolation; with APP_ENV_INJECT (default true)
+# every service also receives them. Bare keys make compose read the values from
+# its own environment, so values never have to be quoted into YAML.
+prepare_env_override() {
+  local name="$1"
+  local override services svc key tmp
+  override="$(app_env_override "${name}")"
+
+  if [[ ${#APP_ENV_KEYS[@]} -eq 0 || "${APP_ENV_INJECT:-true}" != "true" ]]; then
+    rm -f "${override}"
+    COMPOSE_ARGS=(-f "${APP_COMPOSE_FILE}")
+    return
+  fi
+
+  services="$(run_compose -f "${APP_COMPOSE_FILE}" config --services 2>/dev/null)" \
+    || fail "could not list compose services to pass environment variables (is the compose file valid?)"
+
+  tmp="$(mktemp)"
+  {
+    echo "# Generated by panelctl from the app's environment variables. Do not edit."
+    echo "services:"
+    while IFS= read -r svc; do
+      [[ "${svc}" =~ ^[A-Za-z0-9._-]+$ ]] || continue
+      printf '  "%s":\n    environment:\n' "${svc}"
+      for key in "${APP_ENV_KEYS[@]}"; do
+        printf '      - %s\n' "${key}"
+      done
+    done <<<"${services}"
+  } >"${tmp}"
+  install -m 0640 "${tmp}" "${override}"
+  rm -f "${tmp}"
+  COMPOSE_ARGS=(-f "${APP_COMPOSE_FILE}" -f "${override}")
 }
 
 compose_command() {
@@ -239,11 +328,11 @@ run_compose() {
 
   if [[ "${compose}" == *" compose" ]]; then
     local podman_bin="${compose% compose}"
-    "${podman_bin}" compose "$@"
+    env "${APP_ENV_ARGS[@]}" "${podman_bin}" compose "$@"
     return
   fi
 
-  "${compose}" "$@"
+  env "${APP_ENV_ARGS[@]}" "${compose}" "$@"
 }
 
 write_default_compose() {
@@ -337,6 +426,8 @@ cmd_render_route() {
   local route_file
   route_file="$(app_route_file)"
 
+  routes_lock
+
   # Strip any existing block for this app from the aggregate file.
   local tmp
   tmp="$(mktemp)"
@@ -376,6 +467,8 @@ cmd_render_route() {
   } >>"${tmp}"
 
   install -m 0664 -o reudy -g panelroutes "${tmp}" "${route_file}"
+  rm -f "${tmp}"
+  routes_unlock
   log info "rendered route ${route_file}"
 }
 
@@ -387,11 +480,19 @@ cmd_deploy() {
   log info "Starting deployment for app '${name}'"
 
   cmd_render_route "${name}"
+  prepare_env_override "${name}"
 
-  if ! run_compose -f "${APP_COMPOSE_FILE}" up -d --build --remove-orphans 2>&1 | systemd-cat -t panelctl -p info 2>/dev/null; then
+  # Capture compose output so callers (the web UI) can show why a deploy failed,
+  # and still forward it to the journal.
+  local output
+  if ! output="$(run_compose "${COMPOSE_ARGS[@]}" up -d --build --remove-orphans 2>&1)"; then
+    printf '%s\n' "${output}" | systemd-cat -t panelctl -p err 2>/dev/null || true
+    printf '%s\n' "${output}" >&2
     log err "Deployment failed for app '${name}'"
     fail "compose up failed"
   fi
+  printf '%s\n' "${output}" | systemd-cat -t panelctl -p info 2>/dev/null || true
+  printf '%s\n' "${output}"
 
   log info "Successfully deployed app '${name}'"
 }
@@ -403,9 +504,10 @@ cmd_restart() {
 
   log info "Restarting app '${name}'"
 
-  run_compose -f "${APP_COMPOSE_FILE}" down --remove-orphans || fail "compose down failed"
+  run_compose "${COMPOSE_ARGS[@]}" down --remove-orphans || fail "compose down failed"
+  prepare_env_override "${name}"
 
-  if ! run_compose -f "${APP_COMPOSE_FILE}" up -d --build --remove-orphans 2>&1; then
+  if ! run_compose "${COMPOSE_ARGS[@]}" up -d --build --remove-orphans 2>&1; then
     fail "compose up failed during restart"
   fi
 
@@ -417,7 +519,7 @@ cmd_stop() {
   validate_name "${name}"
   load_app "${name}"
 
-  run_compose -f "${APP_COMPOSE_FILE}" down --remove-orphans || fail "compose down failed"
+  run_compose "${COMPOSE_ARGS[@]}" down --remove-orphans || fail "compose down failed"
   log info "stopped app '${name}'"
 }
 
@@ -426,8 +528,8 @@ cmd_status() {
   validate_name "${name}"
   load_app "${name}"
 
-  run_compose -f "${APP_COMPOSE_FILE}" ps --format json 2>/dev/null || \
+  run_compose "${COMPOSE_ARGS[@]}" ps --format json 2>/dev/null || \
-    run_compose -f "${APP_COMPOSE_FILE}" ps 2>/dev/null || \
+    run_compose "${COMPOSE_ARGS[@]}" ps 2>/dev/null || \
     log info "no containers running"
 }
 
@@ -450,7 +552,7 @@ cmd_logs() {
     esac
   done
 
-  run_compose -f "${APP_COMPOSE_FILE}" logs --tail "${tail_lines}" 2>&1 || log info "no logs available"
+  run_compose "${COMPOSE_ARGS[@]}" logs --tail "${tail_lines}" 2>&1 || log info "no logs available"
 }
 
 cmd_validate_compose() {
@@ -458,11 +560,11 @@ cmd_validate_compose() {
   validate_name "${name}"
   load_app "${name}"
 
-  if run_compose -f "${APP_COMPOSE_FILE}" config >/dev/null 2>&1; then
+  if run_compose "${COMPOSE_ARGS[@]}" config >/dev/null 2>&1; then
     log info "compose file is valid"
   else
     local output
-    output="$(run_compose -f "${APP_COMPOSE_FILE}" config 2>&1 || true)"
+    output="$(run_compose "${COMPOSE_ARGS[@]}" config 2>&1 || true)"
     fail "compose validation failed: ${output}"
   fi
 }
@@ -473,19 +575,22 @@ cmd_remove() {
   validate_name "${name}"
   load_app "${name}"
 
-  run_compose -f "${APP_COMPOSE_FILE}" down --remove-orphans 2>/dev/null || true
+  run_compose "${COMPOSE_ARGS[@]}" down --remove-orphans 2>/dev/null || true
 
   # Remove this app's block from the aggregate routes file.
   local route_file
   route_file="$(app_route_file)"
   if [[ -f "${route_file}" ]]; then
+    routes_lock
     local tmp
     tmp="$(mktemp)"
     sed "/^# route:${name}:start$/,/^# route:${name}:end$/d" "${route_file}" >"${tmp}" || true
     install -m 0664 -o reudy -g panelroutes "${tmp}" "${route_file}"
+    rm -f "${tmp}"
+    routes_unlock
   fi
 
-  rm -f "$(app_manifest "${name}")"
+  rm -f "$(app_manifest "${name}")" "$(app_env_file "${name}")"
   rm -rf "${APP_STACK_DIR}"
 
   if [[ "${keep_volumes}" != "--keep-volumes" ]]; then
@@ -551,10 +656,10 @@ cmd_backup() {
 
   # Stop containers before backup for consistency
   local was_running=false
-  if run_compose -f "${APP_COMPOSE_FILE}" ps --format json 2>/dev/null | grep -q '"running"' 2>/dev/null; then
+  if run_compose "${COMPOSE_ARGS[@]}" ps --format json 2>/dev/null | grep -q '"running"' 2>/dev/null; then
     was_running=true
     log info "stopping containers for consistent backup..."
-    run_compose -f "${APP_COMPOSE_FILE}" down 2>/dev/null || true
+    run_compose "${COMPOSE_ARGS[@]}" down 2>/dev/null || true
   fi
 
   (cd "${volume_dir}" && zip -r "${backup_file}" .) || fail "zip failed"
@@ -569,7 +674,7 @@ cmd_backup() {
   # Restart if it was running
   if [[ "${was_running}" == "true" ]]; then
     log info "restarting containers after backup..."
-    run_compose -f "${APP_COMPOSE_FILE}" up -d 2>/dev/null || true
+    run_compose "${COMPOSE_ARGS[@]}" up -d 2>/dev/null || true
   fi
 
   local size
@@ -606,7 +711,7 @@ cmd_volume_clear() {
   load_app "${name}"
 
   log info "clearing volume data for app '${name}'"
-  run_compose -f "${APP_COMPOSE_FILE}" down --remove-orphans 2>/dev/null || true
+  run_compose "${COMPOSE_ARGS[@]}" down --remove-orphans 2>/dev/null || true
 
   local data_dir="${APP_VOLUME_DIR}/data"
   if [[ -d "${data_dir}" ]]; then
@@ -643,7 +748,7 @@ cmd_restore() {
 
   # Stop containers before restore
   log info "stopping containers for restore..."
-  run_compose -f "${APP_COMPOSE_FILE}" down 2>/dev/null || true
+  run_compose "${COMPOSE_ARGS[@]}" down 2>/dev/null || true
 
   # Clear existing volume data and extract backup
   rm -rf "${volume_dir:?}"/*
@@ -671,7 +776,8 @@ cmd_inspect_volumes() {
   if [[ -n "${podman_bin}" ]]; then
     "${podman_bin}" volume ls --filter label=com.docker.compose.project="${name}" --format '{{.Name}}|{{.Mountpoint}}' 2>/dev/null || true
     "${podman_bin}" volume ls --filter label=io.podman.compose.project="${name}" --format '{{.Name}}|{{.Mountpoint}}' 2>/dev/null || true
-  fi | sort -u | grep -v '^$'
+  # grep exits 1 when there are no named volumes; that is not an error.
+  fi | sort -u | grep -v '^$' || true
 }
 
 cmd_list() {