vllm-frontdoor/router/config.py

"""Router configuration: model registry + tunables.

Everything is environment-driven with the defaults from plan v3.3 (see
`.claude/memory/router-front-door-plan.md` sections 4/6).  The `Config`
object is a plain mutable dataclass so tests (and calibration) can override
a single knob without re-reading the environment.

Environment variables (all optional):

  Registry
    VLLM_TEXT_SERVICE   default vllm-text    docker service name (== DNS name)
    VLLM_TEXT_MODEL     default Qwen3.6-35B-A3B-FP8
    VLLM_TEXT_ALIASES   comma list, default "qwen,qwen3,qwen3.6,qwen3.6-35b,
                        qwen3.6-35b-a3b,text,chat,default"
    VLLM_OCR_SERVICE    default vllm-ocr
    VLLM_OCR_MODEL      default OvisOCR2
    VLLM_OCR_ALIASES    default "ocr,ovis,ovisocr,ovis-ocr"
    VLLM_EMBED_SERVICE  default vllm-embed
    VLLM_EMBED_MODEL    default Qwen3-Embedding-8B
    VLLM_EMBED_ALIASES  default "embed,embedding,embeddings,qwen3-embedding,
                        qwen3-embedding-8b"
    VLLM_BACKEND_PORT   default 8000 (container port each vllm service listens on)

  Listeners
    ROUTER_PUBLIC_HOST  default 0.0.0.0
    ROUTER_PUBLIC_PORT  default 8000   (the only public ingress)
    ROUTER_ADMIN_HOST   default 0.0.0.0 (inside the container; the host-side
                        127.0.0.1 restriction comes from compose's
                        "127.0.0.1:8010:8010" port mapping -- docker cannot
                        publish to a container-loopback bind. Set 127.0.0.1
                        when running the router on bare metal.)
    ROUTER_ADMIN_PORT   default 8010   (vllmctl / debugging)

  Proxy (plan 6.2 "Proxy timeouts")
    ROUTER_CONNECT_TIMEOUT   default 5     seconds
    ROUTER_READ_TIMEOUT      default 900   seconds (capped, never disabled)
    ROUTER_POOL_SIZE         default 32    (>= 16 concurrent requests)
    ROUTER_PROBE_TIMEOUT     default 3     seconds for /is_sleeping + /health
    ROUTER_MAX_BODY_BYTES    default 134217728 (128 MiB request body cap)

  Wake-state cache
    ROUTER_STATE_CACHE_TTL   default 2.0 seconds (must stay short: /health
                             returns 200 on a sleeping backend and a proxied
                             request to one HANGS, so a stale "awake" must not
                             bypass the is_sleeping gate for long)

  Depth-aware hold/503 policy (plan 6.2.1; restart deadline from calibration)
    WAKE_HOLD_SLEEP_S        default 30     hold deadline, wake from level 1
                                            (measured wake 2.5-3.8s)
    WAKE_HOLD_OFFLOAD_S      default 180    hold deadline, wake from level 2
    WAKE_HOLD_RESTART_S      default 600    hold deadline, container restarting
                                            (measured text cold boot up to
                                            ~10 min from NFS)
    WAKE_RETRY_AFTER_SLEEP   default 10     Retry-After header value
    WAKE_RETRY_AFTER_OFFLOAD default 60
    WAKE_RETRY_AFTER_RESTART default 600
    WAKE_EST_SLEEP_S         default 6      body estimated_wake_seconds
    WAKE_EST_OFFLOAD_S       default 60
    WAKE_EST_RESTART_S       default 600
    WAKE_ATTEMPTS            default 2      (sequence retried once, plan 6.2.1)
    WAKE_HTTP_TIMEOUT        default 300    per-call timeout inside the sequence
    WAKE_HEALTH_POLL_S       default 2.0    /health poll interval while waking

  Tiered idle (plan 6.3)
    IDLE_SLEEP_MIN           default 15     minutes -> POST /sleep?level=1
    IDLE_OFFLOAD_MIN         default 180    minutes -> POST /sleep?level=2
    IDLE_POLL_SECONDS        default 30     idle-manager scan interval
    IDLE_ENABLED             default 1      (0 disables the background tiering)

  Misc
    ROUTER_LOG_LEVEL         default info
    ROUTER_METRICS           default 0      (1 = also expose /metrics publicly)
    ROUTER_ACCESS_LOG        default 0      (uvicorn access log; off by default,
                             the app logs one structured line per request)
    ROUTER_STATE_FILE        default /state/router-state.json -- persisted wake
                             intent + depth, so a restarted router can tell
                             "awake and ready" from "awake because the previous
                             router died between wake_up and reload_weights".
                             Compose mounts the named volume `router-state`
                             at /state (seeded with image ownership so the
                             non-root user can write it).
                             Set empty to disable persistence.
"""

from __future__ import annotations

import os
from dataclasses import dataclass, field, replace


def _str(name: str, default: str) -> str:
    v = os.environ.get(name)
    return default if v is None or v == "" else v


def _int(name: str, default: int) -> int:
    try:
        return int(_str(name, str(default)))
    except ValueError:
        return default


def _float(name: str, default: float) -> float:
    try:
        return float(_str(name, str(default)))
    except ValueError:
        return default


def _bool(name: str, default: bool) -> bool:
    return _str(name, "1" if default else "0").strip().lower() in ("1", "true", "yes", "on")


def _csv(name: str, default: str) -> tuple[str, ...]:
    return tuple(p.strip() for p in _str(name, default).split(",") if p.strip())


@dataclass(frozen=True)
class ServiceConfig:
    """One registry entry: key -> (docker service, served model name)."""

    key: str
    service: str
    model: str
    aliases: tuple[str, ...] = ()

    @property
    def base_url(self) -> str:
        return f"http://{self.service}:{BACKEND_PORT}"


# Read once at import; the base_url property depends on it.
BACKEND_PORT = _int("VLLM_BACKEND_PORT", 8000)


@dataclass
class DepthPolicy:
    """Hold/retry parameters for one sleep depth (plan 6.2.1 table)."""

    depth: str
    hold_s: float
    retry_after_s: int
    est_wake_s: int
    phrase: str  # "... is waking from <phrase>; retry shortly"

    def error_body(self, model: str) -> dict:
        return {
            "error": {
                "type": "model_waking",
                "code": "model_waking",
                "message": f"Model '{model}' is waking from {self.phrase}; retry shortly",
                "sleep_depth": self.depth,
                "estimated_wake_seconds": self.est_wake_s,
            }
        }


@dataclass
class Config:
    services: dict[str, ServiceConfig]
    # name (lower-cased) -> key, for model resolution
    index: dict[str, str] = field(default_factory=dict)

    public_host: str = "0.0.0.0"
    public_port: int = 8000
    # NOTE: 0.0.0.0 is required *inside the container* -- docker port
    # publishing cannot reach a container-loopback bind.  The "localhost only"
    # guarantee of plan section 4 comes from compose mapping the port as
    # `127.0.0.1:8010:8010` (host side).  On bare metal set
    # ROUTER_ADMIN_HOST=127.0.0.1.
    admin_host: str = "0.0.0.0"
    admin_port: int = 8010

    connect_timeout: float = 5.0
    read_timeout: float = 900.0
    pool_size: int = 32
    probe_timeout: float = 3.0
    max_body_bytes: int = 128 * 1024 * 1024

    state_cache_ttl: float = 2.0

    hold_sleep_s: float = 30.0
    hold_offload_s: float = 180.0
    hold_restart_s: float = 600.0
    retry_after_sleep: int = 10
    retry_after_offload: int = 60
    retry_after_restart: int = 600
    est_sleep_s: int = 6
    est_offload_s: int = 60
    est_restart_s: int = 600
    wake_attempts: int = 2
    wake_http_timeout: float = 300.0
    wake_health_poll_s: float = 2.0

    idle_sleep_min: float = 15.0
    idle_offload_min: float = 180.0
    idle_poll_s: float = 30.0
    idle_enabled: bool = True

    metrics_enabled: bool = False
    log_level: str = "info"
    access_log: bool = False
    # Wake-intent state file (bind-mounted so it survives a router restart).
    state_file: str | None = "/state/router-state.json"

    def policy(self, depth: str) -> DepthPolicy:
        if depth == DEPTH_SLEEPING:
            return DepthPolicy(depth, self.hold_sleep_s, self.retry_after_sleep,
                               self.est_sleep_s, "sleep")
        if depth == DEPTH_RESTARTING:
            return DepthPolicy(depth, self.hold_restart_s, self.retry_after_restart,
                               self.est_restart_s, "a container restart")
        # DEPTH_OFFLOADED and anything unknown: be conservative (plan 6.2.1).
        return DepthPolicy(DEPTH_OFFLOADED, self.hold_offload_s, self.retry_after_offload,
                           self.est_offload_s, "offload")


# Sleep depths tracked by the router.
DEPTH_AWAKE = "awake"
DEPTH_SLEEPING = "sleeping"      # /sleep?level=1  - weights moved to host RAM
DEPTH_OFFLOADED = "offloaded"    # /sleep?level=2  - RAM freed, weights on NFS
DEPTH_RESTARTING = "restarting"  # container down / cold-starting


def load_config() -> Config:
    services: dict[str, ServiceConfig] = {
        "text": ServiceConfig(
            key="text",
            service=_str("VLLM_TEXT_SERVICE", "vllm-text"),
            model=_str("VLLM_TEXT_MODEL", "Qwen3.6-35B-A3B-FP8"),
            aliases=_csv("VLLM_TEXT_ALIASES",
                         "qwen,qwen3,qwen3.6,qwen3.6-35b,qwen3.6-35b-a3b,text,chat,default"),
        ),
        "ocr": ServiceConfig(
            key="ocr",
            service=_str("VLLM_OCR_SERVICE", "vllm-ocr"),
            model=_str("VLLM_OCR_MODEL", "OvisOCR2"),
            aliases=_csv("VLLM_OCR_ALIASES", "ocr,ovis,ovisocr,ovis-ocr"),
        ),
        "embed": ServiceConfig(
            key="embed",
            service=_str("VLLM_EMBED_SERVICE", "vllm-embed"),
            model=_str("VLLM_EMBED_MODEL", "Qwen3-Embedding-8B"),
            aliases=_csv("VLLM_EMBED_ALIASES",
                         "embed,embedding,embeddings,qwen3-embedding,qwen3-embedding-8b"),
        ),
    }

    index: dict[str, str] = {}
    for cfg in services.values():
        names = {cfg.model.lower(), cfg.key.lower(), *(a.lower() for a in cfg.aliases)}
        for name in names:
            index[name] = cfg.key

    return Config(
        services=services,
        index=index,
        public_host=_str("ROUTER_PUBLIC_HOST", "0.0.0.0"),
        public_port=_int("ROUTER_PUBLIC_PORT", 8000),
        admin_host=_str("ROUTER_ADMIN_HOST", "0.0.0.0"),
        admin_port=_int("ROUTER_ADMIN_PORT", 8010),
        connect_timeout=_float("ROUTER_CONNECT_TIMEOUT", 5.0),
        read_timeout=_float("ROUTER_READ_TIMEOUT", 900.0),
        pool_size=_int("ROUTER_POOL_SIZE", 32),
        probe_timeout=_float("ROUTER_PROBE_TIMEOUT", 3.0),
        max_body_bytes=_int("ROUTER_MAX_BODY_BYTES", 128 * 1024 * 1024),
        state_cache_ttl=_float("ROUTER_STATE_CACHE_TTL", 2.0),
        hold_sleep_s=_float("WAKE_HOLD_SLEEP_S", 30.0),
        hold_offload_s=_float("WAKE_HOLD_OFFLOAD_S", 180.0),
        hold_restart_s=_float("WAKE_HOLD_RESTART_S", 600.0),
        retry_after_sleep=_int("WAKE_RETRY_AFTER_SLEEP", 10),
        retry_after_offload=_int("WAKE_RETRY_AFTER_OFFLOAD", 60),
        retry_after_restart=_int("WAKE_RETRY_AFTER_RESTART", 600),
        est_sleep_s=_int("WAKE_EST_SLEEP_S", 6),
        est_offload_s=_int("WAKE_EST_OFFLOAD_S", 60),
        est_restart_s=_int("WAKE_EST_RESTART_S", 600),
        wake_attempts=max(1, _int("WAKE_ATTEMPTS", 2)),
        wake_http_timeout=_float("WAKE_HTTP_TIMEOUT", 300.0),
        wake_health_poll_s=_float("WAKE_HEALTH_POLL_S", 2.0),
        idle_sleep_min=_float("IDLE_SLEEP_MIN", 15.0),
        idle_offload_min=_float("IDLE_OFFLOAD_MIN", 180.0),
        idle_poll_s=_float("IDLE_POLL_SECONDS", 30.0),
        idle_enabled=_bool("IDLE_ENABLED", True),
        metrics_enabled=_bool("ROUTER_METRICS", False),
        log_level=_str("ROUTER_LOG_LEVEL", "info").lower(),
        access_log=_bool("ROUTER_ACCESS_LOG", False),
        state_file=_str("ROUTER_STATE_FILE", "/state/router-state.json") or None,
    )


def clone(cfg: Config) -> Config:
    """Copy for tests."""
    return replace(cfg, services=dict(cfg.services), index=dict(cfg.index))