"""Router configuration: model registry + tunables. Everything is environment-driven with the defaults from plan v3.3 (see `.claude/memory/router-front-door-plan.md` sections 4/6). The `Config` object is a plain mutable dataclass so tests (and calibration) can override a single knob without re-reading the environment. Environment variables (all optional): Registry VLLM_TEXT_SERVICE default vllm-text docker service name (== DNS name) VLLM_TEXT_MODEL default Qwen3.6-35B-A3B-FP8 VLLM_TEXT_ALIASES comma list, default "qwen,qwen3,qwen3.6,qwen3.6-35b, qwen3.6-35b-a3b,text,chat,default" VLLM_OCR_SERVICE default vllm-ocr VLLM_OCR_MODEL default OvisOCR2 VLLM_OCR_ALIASES default "ocr,ovis,ovisocr,ovis-ocr" VLLM_EMBED_SERVICE default vllm-embed VLLM_EMBED_MODEL default Qwen3-Embedding-8B VLLM_EMBED_ALIASES default "embed,embedding,embeddings,qwen3-embedding, qwen3-embedding-8b" VLLM_BACKEND_PORT default 8000 (container port each vllm service listens on) Listeners ROUTER_PUBLIC_HOST default 0.0.0.0 ROUTER_PUBLIC_PORT default 8000 (the only public ingress) ROUTER_ADMIN_HOST default 0.0.0.0 (inside the container; the host-side 127.0.0.1 restriction comes from compose's "127.0.0.1:8010:8010" port mapping -- docker cannot publish to a container-loopback bind. Set 127.0.0.1 when running the router on bare metal.) ROUTER_ADMIN_PORT default 8010 (vllmctl / debugging) Proxy (plan 6.2 "Proxy timeouts") ROUTER_CONNECT_TIMEOUT default 5 seconds ROUTER_READ_TIMEOUT default 900 seconds (capped, never disabled) ROUTER_POOL_SIZE default 32 (>= 16 concurrent requests) ROUTER_PROBE_TIMEOUT default 3 seconds for /is_sleeping + /health ROUTER_MAX_BODY_BYTES default 134217728 (128 MiB request body cap) Wake-state cache ROUTER_STATE_CACHE_TTL default 2.0 seconds (must stay short: /health returns 200 on a sleeping backend and a proxied request to one HANGS, so a stale "awake" must not bypass the is_sleeping gate for long) Depth-aware hold/503 policy (plan 6.2.1; restart deadline from calibration) WAKE_HOLD_SLEEP_S default 30 hold deadline, wake from level 1 (measured wake 2.5-3.8s) WAKE_HOLD_OFFLOAD_S default 180 hold deadline, wake from level 2 WAKE_HOLD_RESTART_S default 600 hold deadline, container restarting (measured text cold boot up to ~10 min from NFS) WAKE_RETRY_AFTER_SLEEP default 10 Retry-After header value WAKE_RETRY_AFTER_OFFLOAD default 60 WAKE_RETRY_AFTER_RESTART default 600 WAKE_EST_SLEEP_S default 6 body estimated_wake_seconds WAKE_EST_OFFLOAD_S default 60 WAKE_EST_RESTART_S default 600 WAKE_ATTEMPTS default 2 (sequence retried once, plan 6.2.1) WAKE_HTTP_TIMEOUT default 300 per-call timeout inside the sequence WAKE_HEALTH_POLL_S default 2.0 /health poll interval while waking Tiered idle (plan 6.3) IDLE_SLEEP_MIN default 15 minutes -> POST /sleep?level=1 IDLE_OFFLOAD_MIN default 180 minutes -> POST /sleep?level=2 IDLE_POLL_SECONDS default 30 idle-manager scan interval IDLE_ENABLED default 1 (0 disables the background tiering) Misc ROUTER_LOG_LEVEL default info ROUTER_METRICS default 0 (1 = also expose /metrics publicly) ROUTER_ACCESS_LOG default 0 (uvicorn access log; off by default, the app logs one structured line per request) ROUTER_STATE_FILE default /state/router-state.json -- persisted wake intent + depth, so a restarted router can tell "awake and ready" from "awake because the previous router died between wake_up and reload_weights". Compose mounts the named volume `router-state` at /state (seeded with image ownership so the non-root user can write it). Set empty to disable persistence. """ from __future__ import annotations import os from dataclasses import dataclass, field, replace def _str(name: str, default: str) -> str: v = os.environ.get(name) return default if v is None or v == "" else v def _int(name: str, default: int) -> int: try: return int(_str(name, str(default))) except ValueError: return default def _float(name: str, default: float) -> float: try: return float(_str(name, str(default))) except ValueError: return default def _bool(name: str, default: bool) -> bool: return _str(name, "1" if default else "0").strip().lower() in ("1", "true", "yes", "on") def _csv(name: str, default: str) -> tuple[str, ...]: return tuple(p.strip() for p in _str(name, default).split(",") if p.strip()) @dataclass(frozen=True) class ServiceConfig: """One registry entry: key -> (docker service, served model name).""" key: str service: str model: str aliases: tuple[str, ...] = () @property def base_url(self) -> str: return f"http://{self.service}:{BACKEND_PORT}" # Read once at import; the base_url property depends on it. BACKEND_PORT = _int("VLLM_BACKEND_PORT", 8000) @dataclass class DepthPolicy: """Hold/retry parameters for one sleep depth (plan 6.2.1 table).""" depth: str hold_s: float retry_after_s: int est_wake_s: int phrase: str # "... is waking from ; retry shortly" def error_body(self, model: str) -> dict: return { "error": { "type": "model_waking", "code": "model_waking", "message": f"Model '{model}' is waking from {self.phrase}; retry shortly", "sleep_depth": self.depth, "estimated_wake_seconds": self.est_wake_s, } } @dataclass class Config: services: dict[str, ServiceConfig] # name (lower-cased) -> key, for model resolution index: dict[str, str] = field(default_factory=dict) public_host: str = "0.0.0.0" public_port: int = 8000 # NOTE: 0.0.0.0 is required *inside the container* -- docker port # publishing cannot reach a container-loopback bind. The "localhost only" # guarantee of plan section 4 comes from compose mapping the port as # `127.0.0.1:8010:8010` (host side). On bare metal set # ROUTER_ADMIN_HOST=127.0.0.1. admin_host: str = "0.0.0.0" admin_port: int = 8010 connect_timeout: float = 5.0 read_timeout: float = 900.0 pool_size: int = 32 probe_timeout: float = 3.0 max_body_bytes: int = 128 * 1024 * 1024 state_cache_ttl: float = 2.0 hold_sleep_s: float = 30.0 hold_offload_s: float = 180.0 hold_restart_s: float = 600.0 retry_after_sleep: int = 10 retry_after_offload: int = 60 retry_after_restart: int = 600 est_sleep_s: int = 6 est_offload_s: int = 60 est_restart_s: int = 600 wake_attempts: int = 2 wake_http_timeout: float = 300.0 wake_health_poll_s: float = 2.0 idle_sleep_min: float = 15.0 idle_offload_min: float = 180.0 idle_poll_s: float = 30.0 idle_enabled: bool = True metrics_enabled: bool = False log_level: str = "info" access_log: bool = False # Wake-intent state file (bind-mounted so it survives a router restart). state_file: str | None = "/state/router-state.json" def policy(self, depth: str) -> DepthPolicy: if depth == DEPTH_SLEEPING: return DepthPolicy(depth, self.hold_sleep_s, self.retry_after_sleep, self.est_sleep_s, "sleep") if depth == DEPTH_RESTARTING: return DepthPolicy(depth, self.hold_restart_s, self.retry_after_restart, self.est_restart_s, "a container restart") # DEPTH_OFFLOADED and anything unknown: be conservative (plan 6.2.1). return DepthPolicy(DEPTH_OFFLOADED, self.hold_offload_s, self.retry_after_offload, self.est_offload_s, "offload") # Sleep depths tracked by the router. DEPTH_AWAKE = "awake" DEPTH_SLEEPING = "sleeping" # /sleep?level=1 - weights moved to host RAM DEPTH_OFFLOADED = "offloaded" # /sleep?level=2 - RAM freed, weights on NFS DEPTH_RESTARTING = "restarting" # container down / cold-starting def load_config() -> Config: services: dict[str, ServiceConfig] = { "text": ServiceConfig( key="text", service=_str("VLLM_TEXT_SERVICE", "vllm-text"), model=_str("VLLM_TEXT_MODEL", "Qwen3.6-35B-A3B-FP8"), aliases=_csv("VLLM_TEXT_ALIASES", "qwen,qwen3,qwen3.6,qwen3.6-35b,qwen3.6-35b-a3b,text,chat,default"), ), "ocr": ServiceConfig( key="ocr", service=_str("VLLM_OCR_SERVICE", "vllm-ocr"), model=_str("VLLM_OCR_MODEL", "OvisOCR2"), aliases=_csv("VLLM_OCR_ALIASES", "ocr,ovis,ovisocr,ovis-ocr"), ), "embed": ServiceConfig( key="embed", service=_str("VLLM_EMBED_SERVICE", "vllm-embed"), model=_str("VLLM_EMBED_MODEL", "Qwen3-Embedding-8B"), aliases=_csv("VLLM_EMBED_ALIASES", "embed,embedding,embeddings,qwen3-embedding,qwen3-embedding-8b"), ), } index: dict[str, str] = {} for cfg in services.values(): names = {cfg.model.lower(), cfg.key.lower(), *(a.lower() for a in cfg.aliases)} for name in names: index[name] = cfg.key return Config( services=services, index=index, public_host=_str("ROUTER_PUBLIC_HOST", "0.0.0.0"), public_port=_int("ROUTER_PUBLIC_PORT", 8000), admin_host=_str("ROUTER_ADMIN_HOST", "0.0.0.0"), admin_port=_int("ROUTER_ADMIN_PORT", 8010), connect_timeout=_float("ROUTER_CONNECT_TIMEOUT", 5.0), read_timeout=_float("ROUTER_READ_TIMEOUT", 900.0), pool_size=_int("ROUTER_POOL_SIZE", 32), probe_timeout=_float("ROUTER_PROBE_TIMEOUT", 3.0), max_body_bytes=_int("ROUTER_MAX_BODY_BYTES", 128 * 1024 * 1024), state_cache_ttl=_float("ROUTER_STATE_CACHE_TTL", 2.0), hold_sleep_s=_float("WAKE_HOLD_SLEEP_S", 30.0), hold_offload_s=_float("WAKE_HOLD_OFFLOAD_S", 180.0), hold_restart_s=_float("WAKE_HOLD_RESTART_S", 600.0), retry_after_sleep=_int("WAKE_RETRY_AFTER_SLEEP", 10), retry_after_offload=_int("WAKE_RETRY_AFTER_OFFLOAD", 60), retry_after_restart=_int("WAKE_RETRY_AFTER_RESTART", 600), est_sleep_s=_int("WAKE_EST_SLEEP_S", 6), est_offload_s=_int("WAKE_EST_OFFLOAD_S", 60), est_restart_s=_int("WAKE_EST_RESTART_S", 600), wake_attempts=max(1, _int("WAKE_ATTEMPTS", 2)), wake_http_timeout=_float("WAKE_HTTP_TIMEOUT", 300.0), wake_health_poll_s=_float("WAKE_HEALTH_POLL_S", 2.0), idle_sleep_min=_float("IDLE_SLEEP_MIN", 15.0), idle_offload_min=_float("IDLE_OFFLOAD_MIN", 180.0), idle_poll_s=_float("IDLE_POLL_SECONDS", 30.0), idle_enabled=_bool("IDLE_ENABLED", True), metrics_enabled=_bool("ROUTER_METRICS", False), log_level=_str("ROUTER_LOG_LEVEL", "info").lower(), access_log=_bool("ROUTER_ACCESS_LOG", False), state_file=_str("ROUTER_STATE_FILE", "/state/router-state.json") or None, ) def clone(cfg: Config) -> Config: """Copy for tests.""" return replace(cfg, services=dict(cfg.services), index=dict(cfg.index))