Fix Portainer Hermes runtime deployment

.env.example

@@ -1,4 +1,6 @@
 HERMES_CONTAINER_USER=0:0
+HERMES_IMAGE=10.0.3.6:4000/zachariahsharma/hermes-control-plane:latest
+HERMES_AGENT_REF=458a94e42568b332e8794ca8fbb8c8e1279160a3
 
 HERMES_SETUP_UI_PORT=7843
 HERMES_PRE_AI_API_PORT=8645
@@ -7,6 +9,7 @@ HERMES_PUBLISHED_BIND_IP=127.0.0.1
 
 HERMES_PRE_AI_PROVIDER=nous
 HERMES_POST_AI_PROVIDER=nous
+HERMES_INTERNAL_API_SERVER_KEY=change-this-to-a-separate-long-random-key
 
 HERMES_HOME_HOST=/opt/hermes-control-plane/hermes
 CODEX_HOME_HOST=/opt/hermes-control-plane/codex

Dockerfile

@@ -1,11 +1,34 @@
 FROM node:20-bookworm-slim
 
+ARG HERMES_AGENT_REF=458a94e42568b332e8794ca8fbb8c8e1279160a3
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      ca-certificates \
+      curl \
+      gcc \
+      git \
+      libffi-dev \
+      python3 \
+      python3-dev \
+      python3-venv \
+      ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git clone --filter=blob:none https://github.com/NousResearch/hermes-agent.git /opt/hermes-agent \
+    && git -C /opt/hermes-agent checkout "$HERMES_AGENT_REF" \
+    && python3 -m venv /opt/hermes-agent/venv \
+    && /opt/hermes-agent/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel \
+    && /opt/hermes-agent/venv/bin/pip install --no-cache-dir -e '/opt/hermes-agent[messaging]' \
+    && /opt/hermes-agent/venv/bin/hermes version
+
 WORKDIR /app
 
 COPY package.json package-lock.json ./
 RUN npm ci --omit=dev
 COPY index.html app.js style.css server.cjs api-gateway.cjs README.md ./
 COPY login.html login.js login.css ./
+COPY docker-entrypoint.sh ./
 COPY lib/ ./lib/
 COPY migrations/ ./migrations/
 
@@ -14,7 +37,8 @@ ENV NODE_ENV=production \
     HERMES_SETUP_UI_PORT=7843 \
     HOME=/home/hermes \
     HERMES_HOME=/home/hermes/.hermes \
-    HERMES_EXE=/home/hermes/.hermes/hermes-agent/venv/bin/hermes \
+    HERMES_EXE=/opt/hermes-agent/venv/bin/hermes \
+    HERMES_DEFAULT_CONFIG=/opt/hermes-agent/cli-config.yaml.example \
     CODEX_HOME=/home/hermes/.codex \
     CLAUDE_CONFIG_DIR=/home/hermes/.claude \
     GEMINI_CONFIG_DIR=/home/hermes/.gemini \
@@ -23,10 +47,12 @@ ENV NODE_ENV=production \
 
 RUN usermod -l hermes -d /home/hermes -m node \
     && groupmod -n hermes node \
-    && chown -R hermes:hermes /app /home/hermes
+    && chmod +x /app/docker-entrypoint.sh \
+    && chown -R hermes:hermes /app /home/hermes /opt/hermes-agent
 
 USER hermes
 
 EXPOSE 7843 8645 8646
 
+ENTRYPOINT ["/app/docker-entrypoint.sh"]
 CMD ["node", "/app/server.cjs"]

README.md

@@ -26,13 +26,11 @@ docker compose up --build -d
 docker compose logs -f
 ```
 
-Open:
+The default stack starts PostgreSQL and the control plane. Open:
 
 ```text
 Admin login:        http://127.0.0.1:7843/login
 Control plane:      http://127.0.0.1:7843
-Pre-Hermes AI API:  http://127.0.0.1:8645/v1/chat/completions
-Post-Hermes AI API: http://127.0.0.1:8646/v1/chat/completions
 ```
 
 Required environment variables for auth and gateway features:
@@ -45,6 +43,9 @@ HERMES_ADMIN_SESSION_TTL_HOURS Session lifetime in hours (default: 8)
 HERMES_ADMIN_COOKIE_SECURE     Set true when admin UI is served only through HTTPS
 HERMES_LOG_RETENTION_DAYS      Audit log retention in days (default: 90)
 HERMES_AUDIT_MAX_BYTES         Max bytes per logged request body (default: 65536)
+HERMES_IMAGE                   Registry image Portainer pulls for app services
+HERMES_AGENT_REF               Pinned Hermes source revision baked into the image
+HERMES_INTERNAL_API_SERVER_KEY Separate internal key required when enabling the post gateway
 ```
 
 ## Admin Login
@@ -61,6 +62,17 @@ If the admin UI is exposed through an HTTPS reverse proxy, set `HERMES_ADMIN_COO
 
 API keys start with `hms_`. Each key is scoped to allow the pre gateway, the post gateway, or both.
 
+The native gateways are intentionally opt-in so missing provider authentication
+cannot break the control plane:
+
+```bash
+# Requires Hermes Nous or xAI authentication in HERMES_HOME_HOST
+docker compose --profile pre-gateway up --build -d
+
+# Starts the native Hermes tool/API gateway
+docker compose --profile post-gateway up --build -d
+```
+
 - **Pre API** (`http://<host>:8645/v1/chat/completions`) — for requests sent before Hermes processing.
 - **Post API** (`http://<host>:8646/v1/chat/completions`) — for requests sent after Hermes processing.
 
@@ -110,11 +122,47 @@ Response content-type is `application/x-ndjson`. Each line is a JSON object with
 
 Use Portainer's Git-backed Stack flow:
 
-1. Push this repo to Git.
+1. Build and push the app image to the registry Portainer can pull.
-2. In Portainer, create a Stack from a Git repository.
+2. Push this repo to Git.
-3. Set the Compose path to `docker-compose.yml`.
+3. In Portainer, create a Stack from a Git repository.
-4. Add the environment variables from `.env.example`, adjusted for your server. At minimum set `POSTGRES_PASSWORD`, `HERMES_ADMIN_USERNAME`, and `HERMES_ADMIN_PASSWORD`.
+4. Set the Compose path to `docker-compose.yml`.
-5. Enable Portainer's auto-update option, either polling or webhook.
+5. Add the environment variables from `.env.example`, adjusted for your server. At minimum set `HERMES_IMAGE`, `POSTGRES_PASSWORD`, `HERMES_ADMIN_USERNAME`, `HERMES_ADMIN_PASSWORD`, and `HERMES_ADMIN_COOKIE_SECURE=true`.
+6. Enable Portainer's auto-update option, either polling or webhook.
+
+Example Gitea package registry image using the internal HTTP registry endpoint.
+Docker image names do not include `http://`; configure the Docker daemon to
+trust `10.0.3.6:4000` as an insecure registry, then use `10.0.3.6:4000/...` in
+the tag.
+
+On the Docker host running Portainer:
+
+```json
+{
+  "insecure-registries": ["10.0.3.6:4000"]
+}
+```
+
+Save that as `/etc/docker/daemon.json`, restart Docker, then verify:
+
+```bash
+sudo systemctl restart docker
+curl -v http://10.0.3.6:4000/v2/
+docker login 10.0.3.6:4000
+```
+
+```bash
+docker login 10.0.3.6:4000
+docker buildx build \
+  --platform linux/amd64 \
+  -t 10.0.3.6:4000/zachariahsharma/hermes-control-plane:latest \
+  --push .
+```
+
+Then set this in Portainer:
+
+```text
+HERMES_IMAGE=10.0.3.6:4000/zachariahsharma/hermes-control-plane:latest
+```
 
 Portainer pulls the latest Git version when it redeploys the Stack. A normal Docker container restart does not pull Git or rebuild the image, so use Portainer's webhook/polling redeploy for updates.
 
@@ -122,6 +170,12 @@ Portainer pulls the latest Git version when it redeploys the Stack. A normal Doc
 
 The compose stack binds published ports to `127.0.0.1` by default via `HERMES_PUBLISHED_BIND_IP`. This prevents clients on the LAN or internet from bypassing Nginx Proxy Manager by calling `http://<docker-host>:7843`, `:8645`, or `:8646` directly.
 
+For `https://hermes.internal.vyntehome.com`, configure that hostname only in
+Nginx Proxy Manager and proxy it to the control plane on port `7843`. Do not set
+an application base-URL environment variable. Browser requests such as
+`/api/status` are intentionally same-origin and resolve to
+`https://hermes.internal.vyntehome.com/api/status` automatically.
+
 Use one of these deployment shapes:
 
 - **NPM on the same Docker host:** keep `HERMES_PUBLISHED_BIND_IP=127.0.0.1` and proxy to `127.0.0.1:7843`, `127.0.0.1:8645`, or `127.0.0.1:8646`.
@@ -130,6 +184,21 @@ Use one of these deployment shapes:
 
 When the public admin URL is HTTPS, also set `HERMES_ADMIN_COOKIE_SECURE=true`.
 
+Recommended Portainer environment for that deployment:
+
+```text
+HERMES_PUBLISHED_BIND_IP=<private Docker-host IP reachable by NPM>
+HERMES_SETUP_UI_PORT=7843
+HERMES_ADMIN_USERNAME=<admin username>
+HERMES_ADMIN_PASSWORD=<long random password>
+HERMES_ADMIN_COOKIE_SECURE=true
+POSTGRES_PASSWORD=<long random password>
+HERMES_HOME_HOST=/opt/hermes-control-plane/hermes
+CODEX_HOME_HOST=/opt/hermes-control-plane/codex
+CLAUDE_HOME_HOST=/opt/hermes-control-plane/claude
+GEMINI_HOME_HOST=/opt/hermes-control-plane/gemini
+```
+
 `HERMES_CONTAINER_USER` defaults to `0:0` so the container can write credentials/config into Portainer-created bind directories. If you pre-create the host directories and `chown` them to UID/GID `1000:1000`, set `HERMES_CONTAINER_USER=1000:1000`.
 
 Recommended persistent host layout on the Portainer host:
@@ -141,22 +210,54 @@ Recommended persistent host layout on the Portainer host:
 /opt/hermes-control-plane/gemini
 ```
 
-The pre/post AI API services both use Hermes' native OpenAI-compatible proxy:
+The Hermes executable is baked into the image at:
 
 ```text
-hermes proxy start --provider <provider> --host 0.0.0.0 --port <port>
+/opt/hermes-agent/venv/bin/hermes
 ```
 
-Set these in Portainer if you want different ports or providers:
+It is intentionally outside `/home/hermes/.hermes`, so mounting the persistent
+Hermes state directory cannot hide or remove the executable. The four host
+directories above contain only mutable state and authentication data.
+
+Provider directories may be empty. For example, a deployment with only Codex
+authentication can leave the Claude and Gemini host directories empty. The
+control plane discovers whichever provider authentication files exist and does
+not require all providers.
+
+On first startup, the container creates missing state directories and seeds a
+default Hermes `config.yaml` only when one does not already exist. Existing
+configuration and authentication files are preserved.
+
+The default Portainer deployment starts only PostgreSQL and the control plane.
+This is deliberate: Codex, Claude, and Gemini auth mounts are optional, and the
+native pre-Hermes proxy supports only providers reported by `hermes proxy providers`.
+This Hermes build currently reports `nous` and `xai`.
+
+```text
+Default services: hermes-postgres, hermes-control-plane
+Pre profile:      hermes-pre-upstream, hermes-pre-api
+Post profile:     hermes-post-upstream, hermes-post-api
+```
+
+Enable optional gateway profiles only after their prerequisites are configured.
+With Docker Compose, use `--profile pre-gateway` or `--profile post-gateway`.
+In a Portainer version that exposes Compose profiles, enable the matching
+profile during stack deployment.
+
+Set these in Portainer when enabling gateways:
 
 ```text
 HERMES_PRE_AI_API_PORT=8645
 HERMES_PRE_AI_PROVIDER=nous
 HERMES_POST_AI_API_PORT=8646
 HERMES_POST_AI_PROVIDER=nous
+HERMES_INTERNAL_API_SERVER_KEY=<separate-long-random-key>
 ```
 
-Run `hermes proxy providers` in the Hermes environment to see supported provider names. This Hermes build reports `nous` and `xai`.
+The public post API accepts the control plane's `hms_` user keys. Internally it
+replaces that header with `HERMES_INTERNAL_API_SERVER_KEY` before forwarding to
+Hermes. Do not reuse the admin password or expose the native upstream ports.
 
 ## Backup
 
@@ -191,17 +292,13 @@ Full request prompts and upstream responses are stored in the audit log for up t
 
 ## Runtime Requirements
 
-The container is Linux. It can only execute a Linux-compatible Hermes install at:
+The container builds its own Linux-compatible Hermes runtime from the pinned
-
+`HERMES_AGENT_REF`. Do not copy or mount a host Hermes virtual environment into
-```text
+the container.
-/home/hermes/.hermes/hermes-agent/venv/bin/hermes
-```
-
-If you are on macOS, your local `~/.hermes` venv contains a macOS Python binary and cannot run inside this Linux container. Use a Linux Hermes home, or run the compose bundle on the Linux Hermes host/LXC.
 
 The default `.env.example` mounts:
 
-- Hermes state from `/opt/hermes-control-plane/hermes`
+- Hermes mutable state from `/opt/hermes-control-plane/hermes`
 - Codex auth/config from `/opt/hermes-control-plane/codex` to `/home/hermes/.codex`
 - Claude auth/config from `/opt/hermes-control-plane/claude` to `/home/hermes/.claude`
 - Gemini auth/config from `/opt/hermes-control-plane/gemini` to `/home/hermes/.gemini`

api-gateway.cjs

@@ -15,6 +15,7 @@ const ROUTE_KIND = process.env.HERMES_API_ROUTE_KIND
 const GATEWAY_HOST = process.env.HERMES_API_GATEWAY_HOST || "0.0.0.0"
 const GATEWAY_PORT = parseInt(process.env.HERMES_API_GATEWAY_PORT || "8080", 10)
 const UPSTREAM_URL = process.env.HERMES_UPSTREAM_URL
+const UPSTREAM_API_KEY = process.env.HERMES_UPSTREAM_API_KEY || ""
 const AUDIT_MAX_BYTES = parseInt(process.env.HERMES_AUDIT_MAX_BYTES || "10485760", 10)
 
 // Validate required env vars
@@ -54,6 +55,15 @@ function filterHeaders(headers) {
   return result
 }
 
+function upstreamRequestHeaders(headers) {
+  const result = filterHeaders(headers)
+  delete result.host
+  if (UPSTREAM_API_KEY) {
+    result.authorization = `Bearer ${UPSTREAM_API_KEY}`
+  }
+  return result
+}
+
 // ─── Upstream request ─────────────────────────────────────────────────────────
 
 /**
@@ -68,9 +78,7 @@ function forwardToUpstream(clientReq) {
     targetUrl.hostname = upstreamBase.hostname
     targetUrl.port = upstreamBase.port
 
-    const forwardHeaders = filterHeaders(clientReq.headers)
+    const forwardHeaders = upstreamRequestHeaders(clientReq.headers)
-    // Remove host — Node's http module will set it correctly
-    delete forwardHeaders["host"]
 
     const options = {
       hostname: targetUrl.hostname,
@@ -307,8 +315,7 @@ async function handleRequest(req, res, pool) {
       targetUrl.hostname = upstreamBase.hostname
       targetUrl.port = upstreamBase.port
 
-      const forwardHeaders = filterHeaders(req.headers)
+      const forwardHeaders = upstreamRequestHeaders(req.headers)
-      delete forwardHeaders["host"]
       // Update content-length if we buffered the body
       if (requestBodyBuffer.length > 0) {
         forwardHeaders["content-length"] = String(requestBodyBuffer.length)

app.js

@@ -64,6 +64,7 @@ function setRoute(name) {
   $$(".dial-item").forEach((el) => el.toggleAttribute("data-active", el.dataset.route === name))
   $$(".pane").forEach((el) => el.toggleAttribute("data-active", el.dataset.pane === name))
   // Lazy loads
+  if (name === "providers") loadProviders()
   if (name === "routing")  loadRouting()
   if (name === "tools")    loadTools()
   if (name === "skills")   loadSkillsRich()

docker-compose.yml

@@ -1,11 +1,15 @@
 x-hermes-build: &hermes-build
   context: .
   dockerfile: Dockerfile
+  args:
+    HERMES_AGENT_REF: ${HERMES_AGENT_REF:-458a94e42568b332e8794ca8fbb8c8e1279160a3}
+
+x-hermes-image: &hermes-image ${HERMES_IMAGE:-10.0.3.6:4000/zachariahsharma/hermes-control-plane:latest}
 
 x-hermes-environment: &hermes-environment
   HOME: /home/hermes
   HERMES_HOME: /home/hermes/.hermes
-  HERMES_EXE: /home/hermes/.hermes/hermes-agent/venv/bin/hermes
+  HERMES_EXE: /opt/hermes-agent/venv/bin/hermes
   CODEX_HOME: /home/hermes/.codex
   CLAUDE_CONFIG_DIR: /home/hermes/.claude
   GEMINI_CONFIG_DIR: /home/hermes/.gemini
@@ -55,7 +59,7 @@ services:
 
   hermes-control-plane:
     build: *hermes-build
-    image: hermes-control-plane:local
+    image: *hermes-image
     user: ${HERMES_CONTAINER_USER:-0:0}
     restart: unless-stopped
     ports:
@@ -82,8 +86,9 @@ services:
       start_period: 10s
 
   hermes-pre-upstream:
+    profiles: ["pre-gateway"]
     build: *hermes-build
-    image: hermes-control-plane:local
+    image: *hermes-image
     user: ${HERMES_CONTAINER_USER:-0:0}
     restart: unless-stopped
     expose:
@@ -104,8 +109,9 @@ services:
       start_period: 20s
 
   hermes-post-upstream:
+    profiles: ["post-gateway"]
     build: *hermes-build
-    image: hermes-control-plane:local
+    image: *hermes-image
     user: ${HERMES_CONTAINER_USER:-0:0}
     restart: unless-stopped
     expose:
@@ -119,6 +125,7 @@ services:
       API_SERVER_ENABLED: "true"
       API_SERVER_HOST: 0.0.0.0
       API_SERVER_PORT: 8642
+      API_SERVER_KEY: ${HERMES_INTERNAL_API_SERVER_KEY:-change-this-internal-hermes-key}
       HERMES_POST_AI_PROVIDER: ${HERMES_POST_AI_PROVIDER:-nous}
     volumes: *hermes-volumes
     healthcheck:
@@ -129,8 +136,9 @@ services:
       start_period: 20s
 
   hermes-pre-api:
+    profiles: ["pre-gateway"]
     build: *hermes-build
-    image: hermes-control-plane:local
+    image: *hermes-image
     user: ${HERMES_CONTAINER_USER:-0:0}
     restart: unless-stopped
     command: ["node", "/app/api-gateway.cjs"]
@@ -157,8 +165,9 @@ services:
       start_period: 10s
 
   hermes-post-api:
+    profiles: ["post-gateway"]
     build: *hermes-build
-    image: hermes-control-plane:local
+    image: *hermes-image
     user: ${HERMES_CONTAINER_USER:-0:0}
     restart: unless-stopped
     command: ["node", "/app/api-gateway.cjs"]
@@ -170,6 +179,7 @@ services:
       HERMES_API_GATEWAY_HOST: 0.0.0.0
       HERMES_API_GATEWAY_PORT: 8646
       HERMES_UPSTREAM_URL: http://hermes-post-upstream:8642
+      HERMES_UPSTREAM_API_KEY: ${HERMES_INTERNAL_API_SERVER_KEY:-change-this-internal-hermes-key}
       HERMES_LOG_RETENTION_DAYS: ${HERMES_LOG_RETENTION_DAYS:-90}
       HERMES_AUDIT_MAX_BYTES: ${HERMES_AUDIT_MAX_BYTES:-10485760}
     depends_on:

docker-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+set -eu
+
+: "${HERMES_HOME:=/home/hermes/.hermes}"
+: "${CODEX_HOME:=/home/hermes/.codex}"
+: "${CLAUDE_CONFIG_DIR:=/home/hermes/.claude}"
+: "${GEMINI_CONFIG_DIR:=/home/hermes/.gemini}"
+: "${HERMES_EXE:=/opt/hermes-agent/venv/bin/hermes}"
+: "${HERMES_DEFAULT_CONFIG:=/opt/hermes-agent/cli-config.yaml.example}"
+
+mkdir -p "$HERMES_HOME" "$CODEX_HOME" "$CLAUDE_CONFIG_DIR" "$GEMINI_CONFIG_DIR"
+
+if [ ! -x "$HERMES_EXE" ]; then
+  echo "startup failed: baked Hermes executable is missing or not executable: $HERMES_EXE" >&2
+  exit 1
+fi
+
+if [ ! -f "$HERMES_HOME/config.yaml" ] && [ -f "$HERMES_DEFAULT_CONFIG" ]; then
+  cp "$HERMES_DEFAULT_CONFIG" "$HERMES_HOME/config.yaml"
+fi
+
+exec "$@"

docs/superpowers/plans/2026-06-08-portainer-hermes-runtime.md

@@ -0,0 +1,54 @@
+# Portainer Hermes Runtime Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Bake Hermes into the image while preserving optional host-mounted provider authentication state.
+
+**Architecture:** Install a pinned Hermes revision at `/opt/hermes-agent`, route all Hermes-backed services through a validating entrypoint, and retain host bind mounts only for mutable state and provider authentication.
+
+**Tech Stack:** Docker, Docker Compose, Bash, Node.js 20, Python 3.11, node:test
+
+---
+
+### Task 1: Deployment Contract Tests
+
+**Files:**
+- Modify: `test/compose-contract.test.cjs`
+- Create: `test/docker-entrypoint.test.cjs`
+
+- [ ] Assert Compose uses `/opt/hermes-agent/venv/bin/hermes`.
+- [ ] Assert the four native state paths remain host bind mounts.
+- [ ] Assert empty state is initialized without requiring provider auth.
+- [ ] Assert existing config is preserved.
+
+### Task 2: Baked Hermes Runtime
+
+**Files:**
+- Modify: `Dockerfile`
+- Create: `docker-entrypoint.sh`
+
+- [ ] Install Python, Git, and build dependencies.
+- [ ] Clone the pinned Hermes revision into `/opt/hermes-agent`.
+- [ ] Create the Hermes virtual environment and install Hermes.
+- [ ] Validate the executable during image build.
+- [ ] Add startup validation and missing-config initialization.
+
+### Task 3: Compose and Documentation
+
+**Files:**
+- Modify: `docker-compose.yml`
+- Modify: `.env.example`
+- Modify: `README.md`
+
+- [ ] Point all services at the baked executable.
+- [ ] Retain configurable host bind mounts for mutable provider state.
+- [ ] Document that provider directories may be empty.
+- [ ] Document Portainer rebuild and NPM forwarding settings.
+
+### Task 4: Verification
+
+- [ ] Run focused deployment tests.
+- [ ] Run `npm test`.
+- [ ] Run `npm run check`.
+- [ ] Build the Docker image.
+- [ ] Run the built image with empty temporary bind mounts and verify `/health`.

docs/superpowers/specs/2026-06-08-portainer-hermes-runtime-design.md

@@ -0,0 +1,60 @@
+# Portainer Hermes Runtime Design
+
+## Goal
+
+Make the Hermes control-plane stack boot reliably in Portainer while preserving
+existing Hermes, Codex, Claude, and Gemini authentication files from host bind
+mounts.
+
+## Architecture
+
+Hermes application code and its Python virtual environment are baked into the
+control-plane image at `/opt/hermes-agent`. Mutable Hermes state remains at
+`/home/hermes/.hermes`, and provider-specific state remains at the native
+`/home/hermes/.codex`, `/home/hermes/.claude`, and `/home/hermes/.gemini`
+paths.
+
+The runtime executable is `/opt/hermes-agent/venv/bin/hermes`. Mounting an
+empty or existing `/home/hermes/.hermes` directory therefore cannot hide the
+executable.
+
+## Host Bind Mount Contract
+
+The stack keeps these configurable host bind mounts:
+
+- `HERMES_HOME_HOST` -> `/home/hermes/.hermes`
+- `CODEX_HOME_HOST` -> `/home/hermes/.codex`
+- `CLAUDE_HOME_HOST` -> `/home/hermes/.claude`
+- `GEMINI_HOME_HOST` -> `/home/hermes/.gemini`
+
+Any provider directory may be empty. Provider discovery remains dynamic and
+must not assume Codex, Claude, or Gemini authentication exists.
+
+## Startup Behavior
+
+Every Hermes-backed service starts through `/app/docker-entrypoint.sh`. The
+entrypoint:
+
+1. Creates missing mutable state directories.
+2. Seeds `config.yaml` only when it does not already exist.
+3. Refuses to start with a clear error if the baked Hermes executable is
+   missing.
+4. Executes the requested control-plane, proxy, or gateway command.
+
+Existing config and authentication files are never overwritten.
+
+## Nginx Proxy Manager
+
+NPM forwards the HTTPS hostname to the control-plane HTTP port. The hostname
+does not participate in Hermes provider-state discovery. Published host ports
+remain bind-address configurable through `HERMES_PUBLISHED_BIND_IP`.
+
+## Verification
+
+- Compose contract tests verify the baked executable path and retained host
+  bind mounts.
+- Entrypoint tests verify empty-state initialization and preservation of
+  existing config.
+- The Docker image build verifies the baked Hermes executable exists and can
+  report its version.
+- Existing application tests continue to pass.

server.cjs

@@ -1706,7 +1706,7 @@ const server = http.createServer(async (req, res) => {
 
     // Always-public routes
     if (key === "GET /health") return h_health(req, res)
-    if (key === "POST /api/admin/login") return h_adminLogin(req, res)
+    if (key === "POST /api/admin/login") return await h_adminLogin(req, res)
     if (key === "GET /login") return serveStatic(req, res)
     if (key === "GET /login.js") return serveStatic(req, res)
     if (key === "GET /login.css") return serveStatic(req, res)
@@ -1714,7 +1714,7 @@ const server = http.createServer(async (req, res) => {
     // Logout is accessible when authenticated (or when auth is off)
     if (key === "POST /api/admin/logout") {
       if (!(await requireAdmin(req, res))) return
-      return h_adminLogout(req, res)
+      return await h_adminLogout(req, res)
     }
 
     // API user management routes (dynamic :id routing — must come before ROUTES lookup)
@@ -1722,13 +1722,13 @@ const server = http.createServer(async (req, res) => {
     if (apiUsersMatch) {
       const [, id, action] = apiUsersMatch
       if (!(await requireAdmin(req, res))) return
-      return handleApiUsersRoute(req, res, id, action)
+      return await handleApiUsersRoute(req, res, id, action)
     }
 
     // All other routes require auth
     if (!(await requireAdmin(req, res))) return
 
-    if (ROUTES[key]) return ROUTES[key](req, res)
+    if (ROUTES[key]) return await ROUTES[key](req, res)
     if (req.method === "GET") return serveStatic(req, res)
     send(res, 404, { error: "not found" })
   } catch (err) {

test/api-gateway.integration.test.cjs

@@ -167,7 +167,7 @@ function gatewayStreamRequest(options) {
 // ─── Tests ────────────────────────────────────────────────────────────────────
 
 test("api-gateway integration", { timeout: 60000 }, async (t) => {
-  await withTestDatabase(t, async ({ pool }) => {
+  await withTestDatabase(t, async ({ pool, schemaName }) => {
     await runMigrations(pool)
 
     // Create test users/keys for the tests
@@ -228,7 +228,9 @@ test("api-gateway integration", { timeout: 60000 }, async (t) => {
     })
     await revokeApiUser(pool, toRevokeUser.id)
 
-    const databaseUrl = process.env.TEST_DATABASE_URL
+    const gatewayDatabaseUrl = new URL(process.env.TEST_DATABASE_URL)
+    gatewayDatabaseUrl.searchParams.set("options", `-c search_path=${schemaName}`)
+    const databaseUrl = gatewayDatabaseUrl.toString()
 
     // Start a fake upstream that returns a simple JSON response
     const jsonUpstream = await startFakeUpstream((req, res) => {
@@ -391,6 +393,44 @@ test("api-gateway integration", { timeout: 60000 }, async (t) => {
       assert.ok(body.usage, "should have usage")
     })
 
+    await t.test("configured upstream API key replaces the client API key", async () => {
+      let receivedAuthorization = null
+      const authenticatedUpstream = await startFakeUpstream((req, res) => {
+        receivedAuthorization = req.headers.authorization
+        req.resume()
+        req.on("end", () => {
+          res.writeHead(200, { "Content-Type": "application/json" })
+          res.end(JSON.stringify({ model: "test", usage: {} }))
+        })
+      })
+      const internalKey = "internal-hermes-api-key"
+      const postGw = await startGateway({
+        databaseUrl,
+        routeKind: "post",
+        upstreamUrl: authenticatedUpstream.url,
+        extraEnv: { HERMES_UPSTREAM_API_KEY: internalKey },
+      })
+
+      try {
+        const { status } = await gatewayRequest({
+          host: postGw.host,
+          port: postGw.port,
+          path: "/v1/chat/completions",
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "Authorization": `Bearer ${postKey}`,
+          },
+          body: JSON.stringify({ model: "test", messages: [] }),
+        })
+        assert.equal(status, 200)
+        assert.equal(receivedAuthorization, `Bearer ${internalKey}`)
+      } finally {
+        await postGw.close()
+        await authenticatedUpstream.close()
+      }
+    })
+
     // ─── Test 7: SSE streaming forwarded ─────────────────────────────────────
     await t.test("SSE streaming: upstream streams text/event-stream, gateway forwards chunks", async () => {
       // Start a streaming upstream

test/compose-contract.test.cjs

@@ -10,7 +10,7 @@ const root = path.join(__dirname, "..")
 function getCompose() {
   try {
     const output = execSync(
-      "docker compose --env-file .env.example config --format json",
+      "docker compose --env-file .env.example --profile pre-gateway --profile post-gateway config --format json",
       { cwd: root, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }
     )
     return JSON.parse(output)
@@ -43,6 +43,11 @@ test("compose config is valid and has correct service structure", (t) => {
   const preApi = services["hermes-pre-api"]
   const postApi = services["hermes-post-api"]
 
+  assert.deepEqual(preUpstream.profiles, ["pre-gateway"], "hermes-pre-upstream should be opt-in")
+  assert.deepEqual(preApi.profiles, ["pre-gateway"], "hermes-pre-api should be opt-in")
+  assert.deepEqual(postUpstream.profiles, ["post-gateway"], "hermes-post-upstream should be opt-in")
+  assert.deepEqual(postApi.profiles, ["post-gateway"], "hermes-post-api should be opt-in")
+
   assert(!preUpstream.ports || preUpstream.ports.length === 0, "hermes-pre-upstream should not publish host ports")
   assert(!postUpstream.ports || postUpstream.ports.length === 0, "hermes-post-upstream should not publish host ports")
 
@@ -51,6 +56,11 @@ test("compose config is valid and has correct service structure", (t) => {
   assert(postApi.ports && postApi.ports.length > 0, "hermes-post-api should publish ports")
 
   const controlPlane = services["hermes-control-plane"]
+  assert.equal(
+    controlPlane.image,
+    "10.0.3.6:4000/zachariahsharma/hermes-control-plane:latest",
+    "Hermes services should default to the internal HTTP registry image"
+  )
   const controlPlanePort = controlPlane.ports?.find((port) => Number(port.target) === 7843)
   const preApiPort = preApi.ports?.find((port) => Number(port.target) === 8645)
   const postApiPort = postApi.ports?.find((port) => Number(port.target) === 8646)
@@ -62,16 +72,41 @@ test("compose config is valid and has correct service structure", (t) => {
   // Post upstream must have API_SERVER_ENABLED=true
   const postUpstreamEnv = postUpstream.environment || {}
   assert.equal(String(postUpstreamEnv.API_SERVER_ENABLED), "true", "hermes-post-upstream API_SERVER_ENABLED must be true")
+  assert(postUpstreamEnv.API_SERVER_KEY, "hermes-post-upstream must have an internal API server key")
 
   // Control plane and gateway services must receive DATABASE_URL
   const controlPlaneEnv = services["hermes-control-plane"].environment || {}
   assert("DATABASE_URL" in controlPlaneEnv, "hermes-control-plane must have DATABASE_URL")
+  assert.equal(
+    controlPlaneEnv.HERMES_EXE,
+    "/opt/hermes-agent/venv/bin/hermes",
+    "Hermes executable must be baked outside the mutable Hermes state mount"
+  )
+
+  const expectedStateTargets = [
+    "/home/hermes/.hermes",
+    "/home/hermes/.codex",
+    "/home/hermes/.claude",
+    "/home/hermes/.gemini",
+  ]
+  for (const serviceName of ["hermes-control-plane", "hermes-pre-upstream", "hermes-post-upstream"]) {
+    const volumes = services[serviceName].volumes || []
+    for (const target of expectedStateTargets) {
+      const mount = volumes.find((volume) => volume.target === target)
+      assert.equal(mount?.type, "bind", `${serviceName} should keep ${target} as a host bind mount`)
+    }
+  }
 
   const preApiEnv = preApi.environment || {}
   assert("DATABASE_URL" in preApiEnv, "hermes-pre-api must have DATABASE_URL")
 
   const postApiEnv = postApi.environment || {}
   assert("DATABASE_URL" in postApiEnv, "hermes-post-api must have DATABASE_URL")
+  assert.equal(
+    postApiEnv.HERMES_UPSTREAM_API_KEY,
+    postUpstreamEnv.API_SERVER_KEY,
+    "post API must authenticate to the native Hermes API with the same internal key"
+  )
 
   // Admin variables only go to control plane
   assert("HERMES_ADMIN_USERNAME" in controlPlaneEnv, "hermes-control-plane must have HERMES_ADMIN_USERNAME")

test/docker-entrypoint.test.cjs

@@ -0,0 +1,71 @@
+"use strict"
+
+const { test } = require("node:test")
+const assert = require("node:assert/strict")
+const fs = require("fs")
+const os = require("os")
+const path = require("path")
+const { spawnSync } = require("child_process")
+
+const root = path.join(__dirname, "..")
+const entrypoint = path.join(root, "docker-entrypoint.sh")
+
+function runEntrypoint(tmp, command) {
+  const hermesHome = path.join(tmp, ".hermes")
+  const codexHome = path.join(tmp, ".codex")
+  const claudeHome = path.join(tmp, ".claude")
+  const geminiHome = path.join(tmp, ".gemini")
+  const defaultConfig = path.join(tmp, "default-config.yaml")
+  const fakeHermes = path.join(tmp, "hermes")
+
+  fs.writeFileSync(defaultConfig, "fallback_providers: []\n")
+  fs.writeFileSync(fakeHermes, "#!/bin/sh\nexit 0\n")
+  fs.chmodSync(fakeHermes, 0o755)
+
+  const result = spawnSync("bash", [entrypoint, ...command], {
+    cwd: root,
+    encoding: "utf-8",
+    env: {
+      ...process.env,
+      HOME: tmp,
+      HERMES_HOME: hermesHome,
+      CODEX_HOME: codexHome,
+      CLAUDE_CONFIG_DIR: claudeHome,
+      GEMINI_CONFIG_DIR: geminiHome,
+      HERMES_EXE: fakeHermes,
+      HERMES_DEFAULT_CONFIG: defaultConfig,
+    },
+  })
+
+  return { result, hermesHome, codexHome, claudeHome, geminiHome }
+}
+
+test("entrypoint initializes empty mutable state without provider authentication", () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "hermes-entrypoint-test-"))
+  try {
+    const state = runEntrypoint(tmp, ["sh", "-c", "printf ready"])
+    assert.equal(state.result.status, 0, state.result.stderr)
+    assert.equal(state.result.stdout, "ready")
+    assert.equal(fs.readFileSync(path.join(state.hermesHome, "config.yaml"), "utf-8"), "fallback_providers: []\n")
+    assert.equal(fs.existsSync(state.codexHome), true)
+    assert.equal(fs.existsSync(state.claudeHome), true)
+    assert.equal(fs.existsSync(state.geminiHome), true)
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true })
+  }
+})
+
+test("entrypoint preserves existing Hermes configuration", () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "hermes-entrypoint-preserve-test-"))
+  try {
+    const hermesHome = path.join(tmp, ".hermes")
+    fs.mkdirSync(hermesHome, { recursive: true })
+    fs.writeFileSync(path.join(hermesHome, "config.yaml"), "existing: true\n")
+
+    const state = runEntrypoint(tmp, ["true"])
+    assert.equal(state.result.status, 0, state.result.stderr)
+    assert.equal(fs.readFileSync(path.join(hermesHome, "config.yaml"), "utf-8"), "existing: true\n")
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true })
+  }
+})

test/server-error-handling.test.cjs

@@ -0,0 +1,66 @@
+"use strict"
+
+const { test } = require("node:test")
+const assert = require("node:assert/strict")
+const fs = require("fs")
+const http = require("http")
+const os = require("os")
+const path = require("path")
+const { spawn } = require("child_process")
+
+const root = path.join(__dirname, "..")
+const serverPath = path.join(root, "server.cjs")
+
+function request(port, requestPath) {
+  return new Promise((resolve, reject) => {
+    http.get({ hostname: "127.0.0.1", port, path: requestPath }, (res) => {
+      let body = ""
+      res.on("data", (chunk) => { body += chunk })
+      res.on("end", () => resolve({ status: res.statusCode, body }))
+    }).on("error", reject)
+  })
+}
+
+async function waitForHealth(port, proc) {
+  const deadline = Date.now() + 5000
+  while (Date.now() < deadline) {
+    if (proc.exitCode !== null) throw new Error(`server exited with ${proc.exitCode}`)
+    try {
+      const response = await request(port, "/health")
+      if (response.status === 200) return
+    } catch {}
+    await new Promise((resolve) => setTimeout(resolve, 100))
+  }
+  throw new Error("server did not become ready")
+}
+
+test("async route errors return 500 without crashing the server", { timeout: 10000 }, async () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "hermes-route-error-test-"))
+  const port = 19745
+  const proc = spawn(process.execPath, [serverPath], {
+    cwd: root,
+    env: {
+      ...process.env,
+      HOME: tmp,
+      HERMES_HOME: path.join(tmp, ".hermes"),
+      HERMES_EXE: path.join(tmp, "missing-hermes"),
+      HERMES_SETUP_UI_HOST: "127.0.0.1",
+      HERMES_SETUP_UI_PORT: String(port),
+      DATABASE_URL: "",
+    },
+    stdio: ["ignore", "pipe", "pipe"],
+  })
+
+  try {
+    await waitForHealth(port, proc)
+    const status = await request(port, "/api/status")
+    assert.equal(status.status, 500)
+    assert.equal(proc.exitCode, null)
+
+    const health = await request(port, "/health")
+    assert.equal(health.status, 200)
+  } finally {
+    proc.kill()
+    fs.rmSync(tmp, { recursive: true, force: true })
+  }
+})

test/ui-contract.test.cjs

@@ -23,6 +23,10 @@ test("app.js contains required functions", (t) => {
   assert.match(app, /downloadApiUserLogs/)
 })
 
+test("providers route reloads provider status", (t) => {
+  assert.match(app, /if \(name === "providers"\)\s+loadProviders\(\)/)
+})
+
 test("style.css contains api-user-table and responsive styles", (t) => {
   assert.match(css, /\.api-user-table/)
   assert.match(css, /@media/)