Initial commit: router front-door vLLM stack
Three always-running vLLM services (text TP=2 GPU0+1, ocr + embed on GPU2, sleep mode) behind a FastAPI router that auto-wakes models on request. Tiered idle (sleep 15 min / offload 3 h), depth-aware 503s with Retry-After, persisted wake-intent recovery, admin API on 127.0.0.1:8010. Routine control via vllmctl is pure HTTP — no docker on the request path. Verified: 91 router unit tests + 15-test E2E on real hardware (measurements in CALIBRATION.md; design record in .claude/memory/router-front-door-plan.md). Old nginx stack files removed before git init; design survives in .claude/memory/sleep-mode-implementation-plan.md. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
293
router/config.py
Normal file
293
router/config.py
Normal file
@@ -0,0 +1,293 @@
|
||||
"""Router configuration: model registry + tunables.
|
||||
|
||||
Everything is environment-driven with the defaults from plan v3.3 (see
|
||||
`.claude/memory/router-front-door-plan.md` sections 4/6). The `Config`
|
||||
object is a plain mutable dataclass so tests (and calibration) can override
|
||||
a single knob without re-reading the environment.
|
||||
|
||||
Environment variables (all optional):
|
||||
|
||||
Registry
|
||||
VLLM_TEXT_SERVICE default vllm-text docker service name (== DNS name)
|
||||
VLLM_TEXT_MODEL default Qwen3.6-35B-A3B-FP8
|
||||
VLLM_TEXT_ALIASES comma list, default "qwen,qwen3,qwen3.6,qwen3.6-35b,
|
||||
qwen3.6-35b-a3b,text,chat,default"
|
||||
VLLM_OCR_SERVICE default vllm-ocr
|
||||
VLLM_OCR_MODEL default OvisOCR2
|
||||
VLLM_OCR_ALIASES default "ocr,ovis,ovisocr,ovis-ocr"
|
||||
VLLM_EMBED_SERVICE default vllm-embed
|
||||
VLLM_EMBED_MODEL default Qwen3-Embedding-8B
|
||||
VLLM_EMBED_ALIASES default "embed,embedding,embeddings,qwen3-embedding,
|
||||
qwen3-embedding-8b"
|
||||
VLLM_BACKEND_PORT default 8000 (container port each vllm service listens on)
|
||||
|
||||
Listeners
|
||||
ROUTER_PUBLIC_HOST default 0.0.0.0
|
||||
ROUTER_PUBLIC_PORT default 8000 (the only public ingress)
|
||||
ROUTER_ADMIN_HOST default 0.0.0.0 (inside the container; the host-side
|
||||
127.0.0.1 restriction comes from compose's
|
||||
"127.0.0.1:8010:8010" port mapping -- docker cannot
|
||||
publish to a container-loopback bind. Set 127.0.0.1
|
||||
when running the router on bare metal.)
|
||||
ROUTER_ADMIN_PORT default 8010 (vllmctl / debugging)
|
||||
|
||||
Proxy (plan 6.2 "Proxy timeouts")
|
||||
ROUTER_CONNECT_TIMEOUT default 5 seconds
|
||||
ROUTER_READ_TIMEOUT default 900 seconds (capped, never disabled)
|
||||
ROUTER_POOL_SIZE default 32 (>= 16 concurrent requests)
|
||||
ROUTER_PROBE_TIMEOUT default 3 seconds for /is_sleeping + /health
|
||||
ROUTER_MAX_BODY_BYTES default 134217728 (128 MiB request body cap)
|
||||
|
||||
Wake-state cache
|
||||
ROUTER_STATE_CACHE_TTL default 2.0 seconds (must stay short: /health
|
||||
returns 200 on a sleeping backend and a proxied
|
||||
request to one HANGS, so a stale "awake" must not
|
||||
bypass the is_sleeping gate for long)
|
||||
|
||||
Depth-aware hold/503 policy (plan 6.2.1; restart deadline from calibration)
|
||||
WAKE_HOLD_SLEEP_S default 30 hold deadline, wake from level 1
|
||||
(measured wake 2.5-3.8s)
|
||||
WAKE_HOLD_OFFLOAD_S default 180 hold deadline, wake from level 2
|
||||
WAKE_HOLD_RESTART_S default 600 hold deadline, container restarting
|
||||
(measured text cold boot up to
|
||||
~10 min from NFS)
|
||||
WAKE_RETRY_AFTER_SLEEP default 10 Retry-After header value
|
||||
WAKE_RETRY_AFTER_OFFLOAD default 60
|
||||
WAKE_RETRY_AFTER_RESTART default 600
|
||||
WAKE_EST_SLEEP_S default 6 body estimated_wake_seconds
|
||||
WAKE_EST_OFFLOAD_S default 60
|
||||
WAKE_EST_RESTART_S default 600
|
||||
WAKE_ATTEMPTS default 2 (sequence retried once, plan 6.2.1)
|
||||
WAKE_HTTP_TIMEOUT default 300 per-call timeout inside the sequence
|
||||
WAKE_HEALTH_POLL_S default 2.0 /health poll interval while waking
|
||||
|
||||
Tiered idle (plan 6.3)
|
||||
IDLE_SLEEP_MIN default 15 minutes -> POST /sleep?level=1
|
||||
IDLE_OFFLOAD_MIN default 180 minutes -> POST /sleep?level=2
|
||||
IDLE_POLL_SECONDS default 30 idle-manager scan interval
|
||||
IDLE_ENABLED default 1 (0 disables the background tiering)
|
||||
|
||||
Misc
|
||||
ROUTER_LOG_LEVEL default info
|
||||
ROUTER_METRICS default 0 (1 = also expose /metrics publicly)
|
||||
ROUTER_ACCESS_LOG default 0 (uvicorn access log; off by default,
|
||||
the app logs one structured line per request)
|
||||
ROUTER_STATE_FILE default /state/router-state.json -- persisted wake
|
||||
intent + depth, so a restarted router can tell
|
||||
"awake and ready" from "awake because the previous
|
||||
router died between wake_up and reload_weights".
|
||||
Compose mounts the named volume `router-state`
|
||||
at /state (seeded with image ownership so the
|
||||
non-root user can write it).
|
||||
Set empty to disable persistence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field, replace
|
||||
|
||||
|
||||
def _str(name: str, default: str) -> str:
|
||||
v = os.environ.get(name)
|
||||
return default if v is None or v == "" else v
|
||||
|
||||
|
||||
def _int(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(_str(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(_str(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _bool(name: str, default: bool) -> bool:
|
||||
return _str(name, "1" if default else "0").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _csv(name: str, default: str) -> tuple[str, ...]:
|
||||
return tuple(p.strip() for p in _str(name, default).split(",") if p.strip())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServiceConfig:
|
||||
"""One registry entry: key -> (docker service, served model name)."""
|
||||
|
||||
key: str
|
||||
service: str
|
||||
model: str
|
||||
aliases: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return f"http://{self.service}:{BACKEND_PORT}"
|
||||
|
||||
|
||||
# Read once at import; the base_url property depends on it.
|
||||
BACKEND_PORT = _int("VLLM_BACKEND_PORT", 8000)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DepthPolicy:
|
||||
"""Hold/retry parameters for one sleep depth (plan 6.2.1 table)."""
|
||||
|
||||
depth: str
|
||||
hold_s: float
|
||||
retry_after_s: int
|
||||
est_wake_s: int
|
||||
phrase: str # "... is waking from <phrase>; retry shortly"
|
||||
|
||||
def error_body(self, model: str) -> dict:
|
||||
return {
|
||||
"error": {
|
||||
"type": "model_waking",
|
||||
"code": "model_waking",
|
||||
"message": f"Model '{model}' is waking from {self.phrase}; retry shortly",
|
||||
"sleep_depth": self.depth,
|
||||
"estimated_wake_seconds": self.est_wake_s,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
services: dict[str, ServiceConfig]
|
||||
# name (lower-cased) -> key, for model resolution
|
||||
index: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
public_host: str = "0.0.0.0"
|
||||
public_port: int = 8000
|
||||
# NOTE: 0.0.0.0 is required *inside the container* -- docker port
|
||||
# publishing cannot reach a container-loopback bind. The "localhost only"
|
||||
# guarantee of plan section 4 comes from compose mapping the port as
|
||||
# `127.0.0.1:8010:8010` (host side). On bare metal set
|
||||
# ROUTER_ADMIN_HOST=127.0.0.1.
|
||||
admin_host: str = "0.0.0.0"
|
||||
admin_port: int = 8010
|
||||
|
||||
connect_timeout: float = 5.0
|
||||
read_timeout: float = 900.0
|
||||
pool_size: int = 32
|
||||
probe_timeout: float = 3.0
|
||||
max_body_bytes: int = 128 * 1024 * 1024
|
||||
|
||||
state_cache_ttl: float = 2.0
|
||||
|
||||
hold_sleep_s: float = 30.0
|
||||
hold_offload_s: float = 180.0
|
||||
hold_restart_s: float = 600.0
|
||||
retry_after_sleep: int = 10
|
||||
retry_after_offload: int = 60
|
||||
retry_after_restart: int = 600
|
||||
est_sleep_s: int = 6
|
||||
est_offload_s: int = 60
|
||||
est_restart_s: int = 600
|
||||
wake_attempts: int = 2
|
||||
wake_http_timeout: float = 300.0
|
||||
wake_health_poll_s: float = 2.0
|
||||
|
||||
idle_sleep_min: float = 15.0
|
||||
idle_offload_min: float = 180.0
|
||||
idle_poll_s: float = 30.0
|
||||
idle_enabled: bool = True
|
||||
|
||||
metrics_enabled: bool = False
|
||||
log_level: str = "info"
|
||||
access_log: bool = False
|
||||
# Wake-intent state file (bind-mounted so it survives a router restart).
|
||||
state_file: str | None = "/state/router-state.json"
|
||||
|
||||
def policy(self, depth: str) -> DepthPolicy:
|
||||
if depth == DEPTH_SLEEPING:
|
||||
return DepthPolicy(depth, self.hold_sleep_s, self.retry_after_sleep,
|
||||
self.est_sleep_s, "sleep")
|
||||
if depth == DEPTH_RESTARTING:
|
||||
return DepthPolicy(depth, self.hold_restart_s, self.retry_after_restart,
|
||||
self.est_restart_s, "a container restart")
|
||||
# DEPTH_OFFLOADED and anything unknown: be conservative (plan 6.2.1).
|
||||
return DepthPolicy(DEPTH_OFFLOADED, self.hold_offload_s, self.retry_after_offload,
|
||||
self.est_offload_s, "offload")
|
||||
|
||||
|
||||
# Sleep depths tracked by the router.
|
||||
DEPTH_AWAKE = "awake"
|
||||
DEPTH_SLEEPING = "sleeping" # /sleep?level=1 - weights moved to host RAM
|
||||
DEPTH_OFFLOADED = "offloaded" # /sleep?level=2 - RAM freed, weights on NFS
|
||||
DEPTH_RESTARTING = "restarting" # container down / cold-starting
|
||||
|
||||
|
||||
def load_config() -> Config:
|
||||
services: dict[str, ServiceConfig] = {
|
||||
"text": ServiceConfig(
|
||||
key="text",
|
||||
service=_str("VLLM_TEXT_SERVICE", "vllm-text"),
|
||||
model=_str("VLLM_TEXT_MODEL", "Qwen3.6-35B-A3B-FP8"),
|
||||
aliases=_csv("VLLM_TEXT_ALIASES",
|
||||
"qwen,qwen3,qwen3.6,qwen3.6-35b,qwen3.6-35b-a3b,text,chat,default"),
|
||||
),
|
||||
"ocr": ServiceConfig(
|
||||
key="ocr",
|
||||
service=_str("VLLM_OCR_SERVICE", "vllm-ocr"),
|
||||
model=_str("VLLM_OCR_MODEL", "OvisOCR2"),
|
||||
aliases=_csv("VLLM_OCR_ALIASES", "ocr,ovis,ovisocr,ovis-ocr"),
|
||||
),
|
||||
"embed": ServiceConfig(
|
||||
key="embed",
|
||||
service=_str("VLLM_EMBED_SERVICE", "vllm-embed"),
|
||||
model=_str("VLLM_EMBED_MODEL", "Qwen3-Embedding-8B"),
|
||||
aliases=_csv("VLLM_EMBED_ALIASES",
|
||||
"embed,embedding,embeddings,qwen3-embedding,qwen3-embedding-8b"),
|
||||
),
|
||||
}
|
||||
|
||||
index: dict[str, str] = {}
|
||||
for cfg in services.values():
|
||||
names = {cfg.model.lower(), cfg.key.lower(), *(a.lower() for a in cfg.aliases)}
|
||||
for name in names:
|
||||
index[name] = cfg.key
|
||||
|
||||
return Config(
|
||||
services=services,
|
||||
index=index,
|
||||
public_host=_str("ROUTER_PUBLIC_HOST", "0.0.0.0"),
|
||||
public_port=_int("ROUTER_PUBLIC_PORT", 8000),
|
||||
admin_host=_str("ROUTER_ADMIN_HOST", "0.0.0.0"),
|
||||
admin_port=_int("ROUTER_ADMIN_PORT", 8010),
|
||||
connect_timeout=_float("ROUTER_CONNECT_TIMEOUT", 5.0),
|
||||
read_timeout=_float("ROUTER_READ_TIMEOUT", 900.0),
|
||||
pool_size=_int("ROUTER_POOL_SIZE", 32),
|
||||
probe_timeout=_float("ROUTER_PROBE_TIMEOUT", 3.0),
|
||||
max_body_bytes=_int("ROUTER_MAX_BODY_BYTES", 128 * 1024 * 1024),
|
||||
state_cache_ttl=_float("ROUTER_STATE_CACHE_TTL", 2.0),
|
||||
hold_sleep_s=_float("WAKE_HOLD_SLEEP_S", 30.0),
|
||||
hold_offload_s=_float("WAKE_HOLD_OFFLOAD_S", 180.0),
|
||||
hold_restart_s=_float("WAKE_HOLD_RESTART_S", 600.0),
|
||||
retry_after_sleep=_int("WAKE_RETRY_AFTER_SLEEP", 10),
|
||||
retry_after_offload=_int("WAKE_RETRY_AFTER_OFFLOAD", 60),
|
||||
retry_after_restart=_int("WAKE_RETRY_AFTER_RESTART", 600),
|
||||
est_sleep_s=_int("WAKE_EST_SLEEP_S", 6),
|
||||
est_offload_s=_int("WAKE_EST_OFFLOAD_S", 60),
|
||||
est_restart_s=_int("WAKE_EST_RESTART_S", 600),
|
||||
wake_attempts=max(1, _int("WAKE_ATTEMPTS", 2)),
|
||||
wake_http_timeout=_float("WAKE_HTTP_TIMEOUT", 300.0),
|
||||
wake_health_poll_s=_float("WAKE_HEALTH_POLL_S", 2.0),
|
||||
idle_sleep_min=_float("IDLE_SLEEP_MIN", 15.0),
|
||||
idle_offload_min=_float("IDLE_OFFLOAD_MIN", 180.0),
|
||||
idle_poll_s=_float("IDLE_POLL_SECONDS", 30.0),
|
||||
idle_enabled=_bool("IDLE_ENABLED", True),
|
||||
metrics_enabled=_bool("ROUTER_METRICS", False),
|
||||
log_level=_str("ROUTER_LOG_LEVEL", "info").lower(),
|
||||
access_log=_bool("ROUTER_ACCESS_LOG", False),
|
||||
state_file=_str("ROUTER_STATE_FILE", "/state/router-state.json") or None,
|
||||
)
|
||||
|
||||
|
||||
def clone(cfg: Config) -> Config:
|
||||
"""Copy for tests."""
|
||||
return replace(cfg, services=dict(cfg.services), index=dict(cfg.index))
|
||||
Reference in New Issue
Block a user