bing/vllm-frontdoor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: router front-door vLLM stack

Browse Source 
Three always-running vLLM services (text TP=2 GPU0+1, ocr + embed on GPU2,
sleep mode) behind a FastAPI router that auto-wakes models on request.
Tiered idle (sleep 15 min / offload 3 h), depth-aware 503s with
Retry-After, persisted wake-intent recovery, admin API on 127.0.0.1:8010.
Routine control via vllmctl is pure HTTP — no docker on the request path.

Verified: 91 router unit tests + 15-test E2E on real hardware
(measurements in CALIBRATION.md; design record in
.claude/memory/router-front-door-plan.md).

Old nginx stack files removed before git init; design survives in
.claude/memory/sleep-mode-implementation-plan.md.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
bing
2026-08-17 10:17:42 +00:00
commit 80eef4ce6a
35 changed files with 6506 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
router/tests/conftest.py Normal file
View File

@@ -0,0 +1,263 @@
"""Shared fixtures + a fake vLLM backend transport (no GPU / docker / network).
The router's outbound httpx clients are built with an injectable transport
(`ServiceManager(cfg, transport=...)`), so a single fake serves all three
backends and records every call the router makes.
"""
from __future__ import annotations
import asyncio
import json
import sys
import urllib.parse
from types import SimpleNamespace
from pathlib import Path
import httpx
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app import build_apps # noqa: E402
from config import clone, load_config # noqa: E402
from services import ServiceManager # noqa: E402
class FakeVLLM(httpx.AsyncBaseTransport):
"""Stands in for vllm-text / vllm-ocr / vllm-embed.
Endpoints: /health, /is_sleeping, /wake_up, /collective_rpc,
/reset_prefix_cache, /sleep?level=, plus /v1/* which echoes what it saw.
"""
def __init__(self) -> None:
self.calls: list[str] = []
self.services: dict[str, dict] = {
"vllm-text": self._svc(),
"vllm-ocr": self._svc(),
"vllm-embed": self._svc(),
}
self.stream_chunks: list[bytes] = []
self.stream_delay: float = 0.0 # per-chunk delay for slow/SSE streams
@staticmethod
def _svc() -> dict:
return {
"reachable": True,
"healthy": True,
"sleeping": False,
"wake_delay": 0.0,
"wake_fails": 0, # number of /wake_up calls to fail (500)
"sleep_fails": 0, # number of /sleep calls to fail (500)
"wake_up_calls": 0,
"reload_calls": 0,
"reset_calls": 0,
"sleep_calls": 0,
"api_calls": 0,
"api_delay": 0.0,
}
# -- helpers -----------------------------------------------------------
@staticmethod
def _response(status: int = 200, *, payload: bytes = b"", content_type: str | None = None):
"""Build a *streamable* response.
`httpx.Response(200, json=...)` would mark the body as already
consumed, which the router's streaming proxy would rightly reject.
"""
async def gen():
if payload:
yield payload
headers = {"content-type": content_type} if content_type else {}
return httpx.Response(status, content=gen(), headers=headers)
def count(self, needle: str) -> int:
return sum(1 for call in self.calls if needle in call)
def last(self, needle: str) -> int:
"""Index of the last matching recorded call (-1 if none)."""
for i in range(len(self.calls) - 1, -1, -1):
if needle in self.calls[i]:
return i
return -1
def set_sleeping(self, key: str, sleeping: bool, level: int = 0) -> None:
svc = self.services[f"vllm-{key}"]
svc["sleeping"] = sleeping
svc["level"] = level
# -- transport ---------------------------------------------------------
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
host = request.url.host or ""
svc = self.services.get(host)
if svc is None or not svc["reachable"]:
raise httpx.ConnectError(f"connection refused: {host}", request=request)
path = request.url.path
params = request.url.params
# record method + full target (path + query) so tests can assert on
# things like "level=1"
target = path + (f"?{params}" if str(params) else "")
self.calls.append(f"{request.method} {host}{target}")
if path == "/is_sleeping":
# Optional hook: report `true` for the next N polls even after a
# wake-up, to model a backend whose wake is still in flight.
if svc.get("hold_sleeping_polls", 0) > 0:
svc["hold_sleeping_polls"] -= 1
return self._response(200, payload=b'{"is_sleeping": true}',
content_type="application/json")
return self._response(200, payload=json.dumps(
{"is_sleeping": bool(svc["sleeping"])}).encode(),
content_type="application/json")
if path == "/health":
# NOTE: like the real vLLM, /health answers 200 even while the
# model is asleep -- readiness must be gated on /is_sleeping.
if not svc["healthy"]:
raise httpx.ConnectError(f"unhealthy: {host}", request=request)
return self._response(200)
if path == "/wake_up":
svc["wake_up_calls"] += 1
if svc["wake_fails"] > 0:
svc["wake_fails"] -= 1
return self._response(500, payload=b"wake failed")
if svc.get("wake_flip_midway") and svc["wake_delay"]:
# Model the real hazard: is_sleeping flips to false *during*
# /wake_up, while reload_weights has not happened yet. A
# router that dies in this window leaves a backend that looks
# perfectly awake and serves garbage (E2E case 12).
await asyncio.sleep(svc["wake_delay"] / 2)
svc["sleeping"] = False
await asyncio.sleep(svc["wake_delay"] / 2)
return self._response(200)
await asyncio.sleep(svc["wake_delay"])
svc["sleeping"] = False
return self._response(200)
if path == "/collective_rpc":
svc["reload_calls"] += 1
return self._response(200)
if path == "/reset_prefix_cache":
svc["reset_calls"] += 1
return self._response(200)
if path == "/sleep":
svc["sleep_calls"] += 1
if svc["sleep_fails"] > 0:
svc["sleep_fails"] -= 1
return self._response(500, payload=b"sleep failed")
svc["sleeping"] = True
svc["level"] = int(params.get("level", "1"))
return self._response(200)
if path.startswith("/v1/"):
svc["api_calls"] += 1
if svc.get("api_delay"):
await asyncio.sleep(svc["api_delay"])
ctype = request.headers.get("content-type", "")
if "text/event-stream" in ctype or self.stream_chunks:
async def gen():
for chunk in self.stream_chunks:
await asyncio.sleep(self.stream_delay)
yield chunk
return httpx.Response(200, content=gen(),
headers={"content-type": "text/event-stream"})
body = (request.content or b"").decode("utf-8", "replace")
return self._response(
200,
payload=json.dumps({
"echo_path": path,
"echo_method": request.method,
"echo_body": body,
"echo_content_type": ctype,
"echo_query": str(request.url.params),
"service": host,
}).encode(),
content_type="application/json",
)
return self._response(404, payload=json.dumps(
{"detail": f"no route {path}"}).encode(), content_type="application/json")
@pytest.fixture
def backend() -> FakeVLLM:
return FakeVLLM()
@pytest.fixture
def cfg():
config = clone(load_config())
config.state_cache_ttl = 0.0 # never trust cache -> deterministic
config.wake_health_poll_s = 0.005
config.idle_enabled = False # idle is exercised via idle_tick()
config.hold_sleep_s = 5.0
config.hold_offload_s = 5.0
config.hold_restart_s = 5.0
return config
@pytest.fixture
async def stack(cfg, backend):
public_app, admin_app, manager = build_apps(cfg, transport=backend)
try:
yield SimpleNamespace(public=public_app, admin=admin_app, manager=manager,
cfg=cfg, backend=backend)
finally:
await manager.proxy_client.aclose()
await manager.ctrl_client.aclose()
@pytest.fixture
async def pub(stack):
transport = httpx.ASGITransport(app=stack.public)
async with httpx.AsyncClient(transport=transport, base_url="http://router.test") as client:
yield client
@pytest.fixture
async def adm(stack):
transport = httpx.ASGITransport(app=stack.admin)
async with httpx.AsyncClient(transport=transport, base_url="http://router.test") as client:
yield client
async def raw_asgi(app, method: str, raw_path: str, *, headers=None, body: bytes = b""):
"""Call the ASGI app with a hand-built scope so the *raw* (percent-encoded)
path reaches the router exactly as a hostile client would send it."""
scope = {
"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"},
"method": method,
"path": urllib.parse.unquote(raw_path),
"raw_path": raw_path.encode("latin-1"),
"query_string": b"",
"headers": [(k.lower().encode(), v.encode()) for k, v in (headers or [])],
"http_version": "1.1", "scheme": "http",
"server": ("router.test", 80), "client": ("127.0.0.1", 1234),
"root_path": "",
}
sent: list[dict] = []
incoming: list[dict] = [{"type": "http.request", "body": body, "more_body": False}]
closed = asyncio.Event()
async def receive():
# Behaves like a real server: hand over the body once, then wait for
# the client to hang up (Starlette's disconnect listener relies on it).
if incoming:
return incoming.pop(0)
await closed.wait()
return {"type": "http.disconnect"}
async def send(message):
sent.append(message)
await app(scope, receive, send)
closed.set()
status = sent[0]["status"]
payload = b"".join(m.get("body", b"") for m in sent if m["type"] == "http.response.body")
return status, payload

120
router/tests/test_admin_api.py Normal file
View File

@@ -0,0 +1,120 @@
"""Admin API surface (this is what `vllmctl` talks to)."""
from __future__ import annotations
import time
from config import DEPTH_AWAKE, DEPTH_OFFLOADED
async def test_status_shape(adm, backend):
backend.set_sleeping("ocr", True, level=1)
r = await adm.get("/admin/status")
assert r.status_code == 200
body = r.json()
assert set(body["services"]) == {"text", "ocr", "embed"}
text = body["services"]["text"]
for field in ("service", "model", "base_url", "reachable", "sleeping",
"depth", "wake_in_progress", "active_requests",
"last_activity_ago_s", "last_wake_latency_s", "last_error"):
assert field in text, field
assert text["model"] == "Qwen3.6-35B-A3B-FP8"
assert text["base_url"] == "http://vllm-text:8000"
assert body["services"]["ocr"]["sleeping"] is True
assert body["services"]["ocr"]["depth"] == "offloaded" # unknown level
assert "http://127.0.0.1:8000/v1" in body["api"]
async def test_wake_endpoint(adm, backend):
backend.set_sleeping("embed", True)
r = await adm.post("/admin/wake/embed")
assert r.status_code == 200
body = r.json()
assert body["ok"] is True
assert body["depth"] == "awake"
assert backend.services["vllm-embed"]["wake_up_calls"] == 1
async def test_wake_endpoint_reports_503_when_it_cannot_wake(adm, backend, stack):
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_fails"] = 5
r = await adm.post("/admin/wake/text")
assert r.status_code == 503
assert r.headers["retry-after"] == "60"
body = r.json()
assert body["ok"] is False
assert body["error"]["sleep_depth"] == "offloaded"
assert body["error"]["estimated_wake_seconds"] == 60
async def test_wake_accepts_model_name_and_case(adm, backend):
backend.set_sleeping("ocr", True)
r = await adm.post("/admin/wake/OvisOCR2")
assert r.status_code == 200
assert backend.services["vllm-ocr"]["wake_up_calls"] == 1
async def test_wake_unknown_key_is_404(adm):
assert (await adm.post("/admin/wake/nope")).status_code == 404
async def test_sleep_level1_and_level2(adm, backend, stack):
r = await adm.post("/admin/sleep/text?level=1")
assert r.status_code == 200
assert r.json()["ok"] is True
assert "level=1" in backend.calls[backend.last("POST vllm-text/sleep")]
assert stack.manager.services["text"].depth == "sleeping"
r = await adm.post("/admin/sleep/text?level=2")
assert r.status_code == 200
assert "level=2" in backend.calls[backend.last("POST vllm-text/sleep")]
assert stack.manager.services["text"].depth == DEPTH_OFFLOADED
async def test_sleep_defaults_to_level1(adm, backend):
r = await adm.post("/admin/sleep/embed")
assert r.status_code == 200
assert "level=1" in backend.calls[backend.last("POST vllm-embed/sleep")]
async def test_sleep_rejects_bad_level(adm):
assert (await adm.post("/admin/sleep/text?level=3")).status_code == 400
assert (await adm.post("/admin/sleep/text?level=abc")).status_code == 400
async def test_sleep_unknown_key_is_404(adm, backend):
assert (await adm.post("/admin/sleep/nope")).status_code == 404
assert backend.calls == []
async def test_sleep_idempotent_when_already_at_depth(adm, backend, stack):
await adm.post("/admin/sleep/ocr?level=1")
backend.calls.clear()
r = await adm.post("/admin/sleep/ocr?level=1")
assert r.status_code == 200
assert r.json()["already"] is True
assert backend.count("POST vllm-ocr/sleep") == 0 # no second POST /sleep
async def test_admin_wake_resets_the_idle_clock(adm, backend, stack):
backend.set_sleeping("text", True)
text = stack.manager.services["text"]
text.last_activity -= 10_000
before = time.monotonic() - text.last_activity
assert (await adm.post("/admin/wake/text")).status_code == 200
assert text.depth == DEPTH_AWAKE
assert (time.monotonic() - text.last_activity) < before
async def test_admin_health(adm):
r = await adm.get("/health")
assert r.status_code == 200
assert r.json()["status"] == "ok"
async def test_admin_health_alias(adm):
"""The documented admin surface says /admin/health."""
assert (await adm.get("/admin/health")).status_code == 200
body = (await adm.get("/admin/health")).json()
assert body["status"] == "ok"
assert "awake" in body

207
router/tests/test_depth_errors.py Normal file
View File

@@ -0,0 +1,207 @@
"""Depth-aware 503 semantics (plan 6.2.1) and never-proxy-half-awake."""
from __future__ import annotations
import asyncio
import httpx
from conftest import raw_asgi
async def _sleep_at_level(stack, level: int) -> None:
result = await stack.manager.sleep_service("text", level, reason="test")
assert result["ok"], result
async def test_503_from_level1_sleep(pub, backend, stack):
await _sleep_at_level(stack, 1)
stack.cfg.hold_sleep_s = 0.05
backend.services["vllm-text"]["wake_delay"] = 0.4
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.headers["retry-after"] == "10"
err = r.json()["error"]
assert err["code"] == "model_waking"
assert err["type"] == "model_waking"
assert err["sleep_depth"] == "sleeping"
assert isinstance(err["estimated_wake_seconds"], int)
assert err["estimated_wake_seconds"] == 6
assert "Qwen3.6-35B-A3B-FP8" in err["message"]
async def test_503_from_level2_offload(pub, backend, stack):
await _sleep_at_level(stack, 2)
stack.cfg.hold_offload_s = 0.05
backend.services["vllm-text"]["wake_delay"] = 0.4
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.headers["retry-after"] == "60"
err = r.json()["error"]
assert err["sleep_depth"] == "offloaded"
assert err["estimated_wake_seconds"] == 60
async def test_503_when_container_restarting(pub, backend, stack):
backend.services["vllm-text"]["reachable"] = False
stack.cfg.hold_restart_s = 0.05
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.headers["retry-after"] == "600"
err = r.json()["error"]
assert err["sleep_depth"] == "restarting"
assert err["estimated_wake_seconds"] == 600
async def test_unknown_depth_is_conservative_offloaded(pub, backend, stack):
"""Router restarted / slept behind our back: depth unknown -> offloaded."""
backend.set_sleeping("ocr", True, level=1) # actually only level 1 asleep
stack.cfg.hold_offload_s = 0.05
backend.services["vllm-ocr"]["wake_delay"] = 0.3
r = await pub.post("/v1/chat/completions", json={"model": "ocr"})
assert r.status_code == 503
err = r.json()["error"]
# conservative: worst-case depth, longest client wait
assert err["sleep_depth"] == "offloaded"
assert r.headers["retry-after"] == "60"
async def test_wake_sequence_retried_once_then_503(pub, backend, stack):
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_fails"] = 2
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.headers["retry-after"] == "60"
err = r.json()["error"]
assert err["code"] == "model_waking"
assert err["sleep_depth"] == "offloaded"
# exactly one retry (plan 6.2.1)
assert backend.count("POST vllm-text/wake_up") == 2
# never proxied to a half-awake backend
assert backend.count("POST vllm-text/v1/") == 0
async def test_never_proxy_before_health_ok(pub, backend):
"""The /v1 call must happen after the wake sequence, never before it.
Requests admitted between wake_up and reload_weights return 200 + garbage,
so the whole sequence has to finish first.
"""
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_delay"] = 0.1
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
svc = backend.services["vllm-text"]
assert svc["wake_up_calls"] == 1
assert svc["reload_calls"] == 1
assert svc["reset_calls"] == 1
calls = backend.calls
idx = {needle: backend.last(needle) for needle in (
"POST vllm-text/wake_up",
"POST vllm-text/collective_rpc",
"POST vllm-text/reset_prefix_cache",
"POST vllm-text/v1/chat/completions",
)}
assert idx["POST vllm-text/v1/chat/completions"] > idx["POST vllm-text/reset_prefix_cache"]
assert idx["POST vllm-text/reset_prefix_cache"] > idx["POST vllm-text/collective_rpc"]
assert idx["POST vllm-text/collective_rpc"] > idx["POST vllm-text/wake_up"]
assert calls[-1] == "POST vllm-text/v1/chat/completions"
async def test_level1_wake_uses_the_fast_path(pub, backend, stack):
"""From level-1 sleep, /wake_up ALONE is enough (calibration 2026-08-17):
bit-identical output at temp 0, and ~20s cheaper than the reload sequence
(23.4s -> 2.5-3.8s on the text model)."""
assert (await stack.manager.sleep_service("text", 1))["ok"] is True
assert stack.manager.services["text"].depth == "sleeping"
backend.calls.clear()
svc = backend.services["vllm-text"]
for field in ("wake_up_calls", "reload_calls", "reset_calls"):
svc[field] = 0
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
assert svc["wake_up_calls"] == 1 # wake_up only ...
assert svc["reload_calls"] == 0 # ... no reload_weights ...
assert svc["reset_calls"] == 0 # ... and no prefix-cache reset
assert backend.count("POST vllm-text/v1/chat/completions") == 1
async def test_unknown_depth_uses_the_full_sequence(pub, backend, stack):
"""Router restarted / slept out of band: unknown depth -> conservative
level-2 treatment, full sequence."""
backend.set_sleeping("text", True) # router depth stays unknown
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
svc = backend.services["vllm-text"]
assert (svc["wake_up_calls"], svc["reload_calls"], svc["reset_calls"]) == (1, 1, 1)
async def test_readiness_is_gated_on_is_sleeping_not_health(pub, backend, stack):
"""/health answers 200 on a sleeping backend, so /is_sleeping is the gate;
nothing is proxied while the backend still reports is_sleeping=true (a
request sent to a sleeping backend hangs instead of erroring)."""
assert (await stack.manager.sleep_service("text", 2))["ok"] is True
backend.calls.clear()
backend.services["vllm-text"]["hold_sleeping_polls"] = 3 # wake "in flight"
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
calls = backend.calls
probes = [i for i, c in enumerate(calls) if c == "GET vllm-text/is_sleeping"]
assert len(probes) >= 4 # kept polling until it flipped
assert calls[-1] == "POST vllm-text/v1/chat/completions" # proxied last
assert backend.services["vllm-text"]["api_calls"] == 1
async def test_backend_dies_between_health_and_proxy(pub, backend, stack):
"""Transport error mid-proxy -> service marked restarting -> depth 503."""
text = backend.services["vllm-text"]
original = backend.handle_async_request
async def flaky(request: httpx.Request) -> httpx.Response:
if request.url.path.startswith("/v1/"):
raise httpx.ConnectError("backend gone", request=request)
return await original(request)
backend.handle_async_request = flaky
stack.cfg.hold_restart_s = 0.05
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.headers["retry-after"] == "600"
assert r.json()["error"]["sleep_depth"] == "restarting"
assert text["wake_up_calls"] == 0
async def test_wake_after_level2_runs_full_sequence(pub, backend):
backend.set_sleeping("embed", True, level=2)
r = await pub.post("/v1/embeddings", json={"input": "hi"})
assert r.status_code == 200
svc = backend.services["vllm-embed"]
assert svc["wake_up_calls"] == 1
assert svc["reload_calls"] == 1 # reload_weights is mandatory after L2
assert svc["reset_calls"] == 1 # prefix cache reset too
assert svc["api_calls"] == 1
async def test_error_body_is_openai_shaped(pub, backend, stack):
await _sleep_at_level(stack, 1)
stack.cfg.hold_sleep_s = 0.01
backend.services["vllm-text"]["wake_delay"] = 0.2
r = await pub.post("/v1/chat/completions", json={"model": "text"})
payload = r.json()
assert set(payload) == {"error"}
assert set(payload["error"]) == {
"type", "code", "message", "sleep_depth", "estimated_wake_seconds"
}
assert r.headers["content-type"].startswith("application/json")
async def test_depth_survives_raw_traversal_requests(stack, backend):
"""Traversal requests are rejected before any backend contact."""
backend.set_sleeping("text", True)
for raw in ("/v1/../sleep", "/v1%2f..%2fsleep", "//sleep", "/v1/../../wake_up"):
status, _ = await raw_asgi(stack.public, "POST", raw)
assert status == 404, raw
assert backend.calls == []

195
router/tests/test_idle.py Normal file
View File

@@ -0,0 +1,195 @@
"""Tiered idle management: thresholds, lock races, active-request guard."""
from __future__ import annotations
import asyncio
import time
from config import DEPTH_AWAKE, DEPTH_OFFLOADED, DEPTH_SLEEPING
def _age(svc, seconds: float) -> None:
svc.last_activity = time.monotonic() - seconds
async def test_idle_level1_after_threshold(stack, backend):
stack.cfg.idle_sleep_min = 15 / 60.0 # 15 s in "minutes"
stack.cfg.idle_offload_min = 180 / 60.0
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 20)
await stack.manager.idle_tick()
assert backend.count("POST vllm-text/sleep") == 1
assert "level=1" in backend.calls[backend.last("POST vllm-text/sleep")]
assert text.depth == DEPTH_SLEEPING
async def test_idle_escalates_to_level2(stack, backend):
stack.cfg.idle_sleep_min = 15 / 60.0
stack.cfg.idle_offload_min = 180 / 60.0
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 400)
await stack.manager.idle_tick()
assert "level=2" in backend.calls[backend.last("POST vllm-text/sleep")]
assert text.depth == DEPTH_OFFLOADED
async def test_level1_service_escalates_to_level2(stack, backend):
"""Already napping (level 1) and still idle -> escalate to offload.
A direct POST /sleep?level=2 on a level-1-sleeping backend is a
well-behaved NO-OP that retains the host-RAM copy (calibration
2026-08-17), so the escalation must wake into RAM first and then offload.
"""
stack.cfg.idle_sleep_min = 1 / 60.0
stack.cfg.idle_offload_min = 5 / 60.0
text = stack.manager.services["text"]
assert (await stack.manager.sleep_service("text", 1))["ok"] is True
backend.calls.clear()
svc = backend.services["vllm-text"]
svc["wake_up_calls"] = svc["sleep_calls"] = 0
_age(text, 400)
await stack.manager.idle_tick()
assert text.depth == DEPTH_OFFLOADED
assert svc["wake_up_calls"] == 1 # wake into RAM ...
assert svc["sleep_calls"] == 1 # ... then offload
assert backend.last("POST vllm-text/sleep?level=2") > backend.last("POST vllm-text/wake_up")
async def test_offload_from_awake_is_direct(stack, backend):
"""Only from depth 'awake' can level 2 be entered directly."""
backend.services["vllm-embed"]["wake_up_calls"] = 0
result = await stack.manager.sleep_service("embed", 2)
assert result["ok"] is True
assert backend.services["vllm-embed"]["wake_up_calls"] == 0
assert backend.count("POST vllm-embed/sleep?level=2") == 1
assert stack.manager.services["embed"].depth == DEPTH_OFFLOADED
async def test_escalation_failure_is_reported(stack, backend):
assert (await stack.manager.sleep_service("ocr", 1))["ok"] is True
backend.services["vllm-ocr"]["wake_fails"] = 1
backend.calls.clear()
result = await stack.manager.sleep_service("ocr", 2)
assert result["ok"] is False
assert result["reason"] == "escalation_failed"
assert backend.count("POST vllm-ocr/sleep") == 0
assert stack.manager.services["ocr"].depth == DEPTH_SLEEPING
async def test_offloaded_service_is_left_alone(stack, backend):
stack.cfg.idle_offload_min = 1 / 60.0
text = stack.manager.services["text"]
text.depth = DEPTH_OFFLOADED
_age(text, 10_000)
await stack.manager.idle_tick()
assert backend.calls == []
async def test_active_requests_block_idle_sleep(stack, backend):
stack.cfg.idle_sleep_min = 1 / 60.0
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 600)
manager = stack.manager
manager.begin_request(text) # long generation in flight
try:
await manager.idle_tick()
assert backend.calls == []
finally:
manager.end_request(text)
async def test_stream_completes_then_idle_can_sleep(stack, backend, pub):
"""end_request (background task of the streamed response) re-opens sleep."""
stack.cfg.idle_sleep_min = 1 / 60.0
text = stack.manager.services["text"]
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
assert text.active_requests == 0 # released after the body drained
_age(text, 600)
await stack.manager.idle_tick()
assert backend.count("POST vllm-text/sleep") == 1
async def test_race_last_activity_refreshed_under_lock(stack, backend):
"""A request landed between the threshold check and the locked re-check."""
stack.cfg.idle_sleep_min = 1 / 60.0 # threshold = 60 s
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 600) # what the idle scan saw
# ... and then a request arrived, refreshing last_activity *now*
text.last_activity = time.monotonic()
result = await stack.manager.sleep_service("text", 1, reason="idle", min_idle_s=60.0)
assert result["ok"] is False
assert result["reason"] == "activity_resumed"
assert backend.count("POST vllm-text/sleep") == 0
async def test_race_active_request_seen_under_lock(stack, backend):
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 600)
text.active_requests = 2 # arrived while we were scanning
result = await stack.manager.sleep_service("text", 1, reason="idle", min_idle_s=60.0)
assert result["ok"] is False
assert result["reason"] == "active_requests"
assert backend.calls == []
text.active_requests = 0
async def test_request_at_the_moment_the_timer_expires(pub, backend, stack):
"""E2E case 11 in miniature: the request wins, no sleep mid-flight."""
stack.cfg.idle_sleep_min = 1 / 60.0
stack.cfg.idle_offload_min = 5 / 60.0
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 600)
async def tick_soon():
await asyncio.sleep(0) # run after the request started
return await stack.manager.idle_tick()
tick, response = await asyncio.gather(tick_soon(), pub.post(
"/v1/chat/completions", json={"model": "text"}))
assert response.status_code == 200
assert backend.count("POST vllm-text/sleep") == 0
async def test_admin_sleep_refuses_when_active(adm, stack, backend):
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
stack.manager.begin_request(text)
r = await adm.post("/admin/sleep/text?level=1")
assert r.status_code == 409
assert r.json()["reason"] == "active_requests"
assert backend.count("/sleep") == 0
stack.manager.end_request(text)
async def test_idle_loop_runs_in_background_when_started(stack, backend):
stack.cfg.idle_enabled = True
stack.cfg.idle_poll_s = 0.01
stack.cfg.idle_sleep_min = 1 / 60.0 # 60 s
stack.cfg.idle_offload_min = 1000.0 # far away: expect the level-1 tier
text = stack.manager.services["text"]
text.depth = DEPTH_AWAKE
_age(text, 600)
stack.manager.start()
try:
for _ in range(100):
if backend.count("POST vllm-text/sleep"):
break
await asyncio.sleep(0.01)
finally:
await stack.manager.stop()
assert backend.count("POST vllm-text/sleep") == 1
assert "level=1" in backend.calls[backend.last("POST vllm-text/sleep")]

160
router/tests/test_model_resolution.py Normal file
View File

@@ -0,0 +1,160 @@
"""Model resolution: JSON / multipart / defaults / embeddings / unknown."""
from __future__ import annotations
import json
TEXT = "Qwen3.6-35B-A3B-FP8"
OCR = "OvisOCR2"
EMBED = "Qwen3-Embedding-8B"
async def test_json_model_exact(pub, backend):
r = await pub.post("/v1/chat/completions", json={"model": OCR, "messages": []})
assert r.status_code == 200
assert r.json()["service"] == "vllm-ocr"
async def test_json_model_case_insensitive(pub):
r = await pub.post("/v1/chat/completions", json={"model": "qwen3.6-35b-a3b-fp8"})
assert r.status_code == 200
assert r.json()["service"] == "vllm-text"
async def test_alias_matches(pub):
r = await pub.post("/v1/chat/completions", json={"model": "ocr"})
assert r.status_code == 200
assert r.json()["service"] == "vllm-ocr"
r = await pub.post("/v1/chat/completions", json={"model": "Embedding"})
assert r.json()["service"] == "vllm-embed"
async def test_missing_model_defaults_to_text_on_chat(pub):
r = await pub.post("/v1/chat/completions", json={"messages": [{"role": "user", "content": "hi"}]})
assert r.status_code == 200
assert r.json()["service"] == "vllm-text"
async def test_missing_model_defaults_to_text_on_completions(pub):
r = await pub.post("/v1/completions", json={"prompt": "hi"})
assert r.status_code == 200
assert r.json()["service"] == "vllm-text"
async def test_missing_model_is_400_elsewhere(pub, backend):
r = await pub.post("/v1/rerank", json={"query": "hi"})
assert r.status_code == 400
assert r.json()["error"]["code"] == "missing_model"
assert backend.count("/v1/rerank") == 0
async def test_invalid_json_is_400(pub, backend):
r = await pub.post("/v1/chat/completions",
content=b"{not json",
headers={"content-type": "application/json"})
assert r.status_code == 400
assert backend.count("/v1/") == 0
async def test_embeddings_always_routes_to_embed(pub):
r = await pub.post("/v1/embeddings", json={"input": "hello"})
assert r.status_code == 200
assert r.json()["service"] == "vllm-embed"
# ... even when the body names a different model
r = await pub.post("/v1/embeddings", json={"input": "hello", "model": TEXT})
assert r.status_code == 200
assert r.json()["service"] == "vllm-embed"
async def test_unknown_model_404_and_no_wake(pub, backend):
backend.set_sleeping("ocr", True)
r = await pub.post("/v1/chat/completions", json={"model": "gpt-4o"})
assert r.status_code == 404
body = r.json()["error"]
assert body["code"] == "model_not_found"
assert body["type"] == "invalid_request_error"
assert body["param"] == "model"
# no probe, no wake, no proxy
assert backend.calls == []
async def test_multipart_with_model_field(pub):
r = await pub.post(
"/v1/chat/completions",
data={"model": OCR},
files={"image": ("page.png", b"PNGDATA" * 64, "image/png")},
)
assert r.status_code == 200
echo = r.json()
assert echo["service"] == "vllm-ocr"
# raw body forwarded unchanged: the file bytes and the boundary survive
assert "PNGDATA" * 64 in echo["echo_body"]
assert echo["echo_content_type"].startswith("multipart/form-data; boundary=")
async def test_multipart_without_model_defaults_to_ocr(pub):
r = await pub.post(
"/v1/chat/completions",
files={"image": ("page.png", b"PNGDATA", "image/png")},
)
assert r.status_code == 200
assert r.json()["service"] == "vllm-ocr"
async def test_multipart_unknown_model_404(pub, backend):
r = await pub.post(
"/v1/chat/completions",
data={"model": "nope"},
files={"image": ("page.png", b"x", "image/png")},
)
assert r.status_code == 404
assert r.json()["error"]["code"] == "model_not_found"
assert backend.calls == []
async def test_multipart_model_extraction_ignores_file_parts(pub):
# a file part literally named "model" must not be read as the model field
r = await pub.post(
"/v1/chat/completions",
data={"prompt": "hi"},
files={"model": ("fake.json", b"NOT-A-MODEL-NAME", "application/octet-stream")},
)
# no usable model field -> multipart default on the chat path is OCR
assert r.status_code == 200
assert r.json()["service"] == "vllm-ocr"
assert "NOT-A-MODEL-NAME" in r.json()["echo_body"]
async def test_model_in_path_for_models_detail(pub):
r = await pub.get(f"/v1/models/{OCR}")
assert r.status_code == 200
assert r.json()["service"] == "vllm-ocr"
r = await pub.get("/v1/models/does-not-exist")
assert r.status_code == 404
assert r.json()["error"]["code"] == "model_not_found"
async def test_body_and_headers_forwarded(pub):
payload = {"model": TEXT, "messages": [{"role": "user", "content": "hi"}], "stream": False}
r = await pub.post("/v1/chat/completions", json=payload,
headers={"Authorization": "Bearer x", "x-custom": "1"})
assert r.status_code == 200
echo = r.json()
assert echo["echo_path"] == "/v1/chat/completions"
assert echo["echo_method"] == "POST"
assert json.loads(echo["echo_body"]) == payload
async def test_query_string_forwarded(pub):
r = await pub.post("/v1/chat/completions?foo=bar&baz=1", json={"model": TEXT})
assert r.status_code == 200
assert r.json()["echo_query"] == "foo=bar&baz=1"
async def test_streaming_passthrough(pub, backend):
backend.stream_chunks = [b"data: {\"a\":1}\n\n", b"data: {\"a\":2}\n\n", b"data: [DONE]\n\n"]
r = await pub.post("/v1/chat/completions", json={"model": TEXT, "stream": True})
assert r.status_code == 200
assert r.headers["content-type"].startswith("text/event-stream")
assert r.text == "".join(chunk.decode() for chunk in backend.stream_chunks)
backend.stream_chunks = []

146
router/tests/test_paths.py Normal file
View File

@@ -0,0 +1,146 @@
"""Allowlist: only /v1/*, /health (and /metrics when enabled) are public."""
from __future__ import annotations
import httpx
from app import build_apps
from conftest import raw_asgi
DEV_ENDPOINTS = [
"/sleep",
"/wake_up",
"/collective_rpc",
"/reset_prefix_cache",
"/is_sleeping",
]
ADMIN_PATHS = ["/admin/status", "/admin/wake/text", "/admin/sleep/text?level=1"]
async def test_dev_endpoints_404_on_public(pub, backend):
for path in DEV_ENDPOINTS:
r = await pub.post(path)
assert r.status_code == 404, path
assert r.json()["error"]["code"] == "not_found"
for path in DEV_ENDPOINTS:
r = await pub.get(path)
assert r.status_code == 404, path
assert backend.calls == []
async def test_admin_not_on_public(pub, backend):
for path in ADMIN_PATHS:
r = await pub.request("POST", path)
assert r.status_code == 404, path
r = await pub.get(path)
assert r.status_code == 404, path
assert backend.calls == []
async def test_docs_and_openapi_404(pub):
for path in ("/docs", "/redoc", "/openapi.json", "/"):
r = await pub.get(path)
assert r.status_code == 404, path
async def test_dotted_paths_404(pub):
for path in ("/v1/../sleep", "/v1/..%2fsleep", "/health/../sleep", "/./sleep"):
r = await pub.post(path)
assert r.status_code == 404, path
async def test_raw_traversal_404(stack, backend):
"""Percent-encoded traversal that a normal client would normalise."""
cases = [
("POST", "/v1%2f..%2fsleep"),
("POST", "/v1%2F..%2Fsleep"),
("GET", "/v1/%2e%2e/wake_up"),
("POST", "/v1/%2e%2e%2fcollective_rpc"),
("POST", "/v1//../sleep"),
("GET", "/v1/models/../../is_sleeping"),
]
for method, raw in cases:
status, body = await raw_asgi(stack.public, method, raw,
headers=[("content-type", "application/json")],
body=b"{}")
assert status == 404, (method, raw, body)
assert backend.calls == []
async def test_normalised_traversal_still_reaches_v1(stack, backend):
"""A traversal that lands inside /v1 must still work (not over-block)."""
status, body = await raw_asgi(
stack.public, "POST", "/v1/../v1/chat/completions",
headers=[("content-type", "application/json")],
body=b'{"model": "text"}',
)
assert status == 200, body
assert backend.count("POST vllm-text/v1/chat/completions") == 1
async def test_public_health(pub, backend):
r = await pub.get("/health")
assert r.status_code == 200
body = r.json()
assert body["status"] == "ok"
assert set(body["services"]) == {"text", "ocr", "embed"}
# liveness must not depend on backend state
backend.services["vllm-text"]["reachable"] = False
r = await pub.get("/health")
assert r.status_code == 200
async def test_public_models_lists_all_three(pub, backend):
r = await pub.get("/v1/models")
assert r.status_code == 200
body = r.json()
assert body["object"] == "list"
assert [m["id"] for m in body["data"]] == [
"Qwen3.6-35B-A3B-FP8", "OvisOCR2", "Qwen3-Embedding-8B",
]
# answered by the router itself, no backend contact
assert backend.calls == []
async def test_metrics_disabled_by_default(pub):
assert (await pub.get("/metrics")).status_code == 404
async def test_metrics_enabled_when_configured(stack, pub):
stack.cfg.metrics_enabled = True
r = await pub.get("/metrics")
assert r.status_code == 200
assert "vllm_router_service_depth" in r.text
async def test_unknown_http_method_on_v1(stack):
status, _ = await raw_asgi(stack.public, "PROPFIND", "/v1/chat/completions")
assert status in (404, 405)
async def test_admin_listener_is_a_separate_app(stack):
"""The admin app answers /admin/*; the public app must not."""
status, body = await raw_asgi(stack.admin, "GET", "/admin/status")
assert status == 200
status, _ = await raw_asgi(stack.public, "GET", "/admin/status")
assert status == 404
async def test_request_body_cap(cfg, backend):
"""ROUTER_MAX_BODY_BYTES must be wired as middleware (413, not OOM)."""
cfg.max_body_bytes = 64
public_app, _admin, manager = build_apps(cfg, transport=backend)
ok_body = b'{"model": "text", "messages": []}'
assert len(ok_body) < 64
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=public_app),
base_url="http://router.test") as client:
r = await client.post("/v1/chat/completions", content=ok_body,
headers={"content-type": "application/json"})
assert r.status_code == 200
r = await client.post("/v1/chat/completions", content=ok_body + b" " * 512,
headers={"content-type": "application/json"})
assert r.status_code == 413
finally:
await manager.proxy_client.aclose()
await manager.ctrl_client.aclose()

101
router/tests/test_shutdown_and_cli.py Normal file
View File

@@ -0,0 +1,101 @@
"""Shutdown behaviour (E2E case 12) and the vllmctl CLI glue."""
from __future__ import annotations
import asyncio
import json
import os
import subprocess
import pytest
from conftest import raw_asgi
from routing import ShutdownGuardMiddleware
async def test_shutdown_guard_returns_503_when_not_started(stack):
"""uvicorn cancels in-flight tasks after timeout_graceful_shutdown; that
CancelledError must become a depth-aware 503, not uvicorn's bare 500."""
stack.manager.begin_shutdown()
async def app(scope, receive, send):
raise asyncio.CancelledError()
guard = ShutdownGuardMiddleware(app, stack.manager)
status, body = await raw_asgi(guard, "POST", "/v1/chat/completions")
assert status == 503
payload = json.loads(body)
assert payload["error"]["code"] == "router_shutting_down"
assert payload["error"]["retry_after_seconds"] == 5
async def test_shutdown_guard_passes_through_when_not_shutting_down(stack):
class Boom(RuntimeError):
pass
async def app(scope, receive, send):
raise Boom("nope")
guard = ShutdownGuardMiddleware(app, stack.manager)
with pytest.raises(Boom):
await raw_asgi(guard, "POST", "/v1/chat/completions")
async def test_shutdown_guard_cannot_unsend_a_started_response(stack):
"""A response that already started streaming is closed, not replaced."""
stack.manager.begin_shutdown()
async def app(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"partial"})
raise asyncio.CancelledError()
guard = ShutdownGuardMiddleware(app, stack.manager)
with pytest.raises(asyncio.CancelledError):
await raw_asgi(guard, "POST", "/v1/chat/completions")
async def test_shutdown_guard_forwards_normal_requests(stack):
async def app(scope, receive, send):
await send({"type": "http.response.start", "status": 204, "headers": []})
await send({"type": "http.response.body", "body": b""})
guard = ShutdownGuardMiddleware(app, stack.manager)
status, _ = await raw_asgi(guard, "GET", "/health")
assert status == 204
def test_vllmctl_service_name_mapping():
"""`logs router` / `restart router` must map to the compose service
`router`; the model services map to vllm-*."""
script = (
"source /data/home/renbaibing/vllm/vllmctl >/dev/null 2>&1; "
'for n in router vllm-router Router text vllm-text TEXT ocr OvisOCR2 '
"embed Qwen3-Embedding-8B bogus; do "
'printf "%s:%s\\n" "$n" "$(docker_service_for "$n" || echo ERR)"; done'
)
out = subprocess.run(["bash", "-c", script], capture_output=True, text=True,
timeout=30)
assert out.returncode == 0, out.stderr
mapping = dict(line.split(":", 1) for line in out.stdout.strip().splitlines())
assert mapping["router"] == "router"
assert mapping["vllm-router"] == "router"
assert mapping["Router"] == "router"
assert mapping["text"] == "vllm-text"
assert mapping["vllm-text"] == "vllm-text"
assert mapping["TEXT"] == "vllm-text"
assert mapping["ocr"] == "vllm-ocr"
assert mapping["OvisOCR2"] == "vllm-ocr"
assert mapping["embed"] == "vllm-embed"
assert mapping["Qwen3-Embedding-8B"] == "vllm-embed"
assert mapping["bogus"] == "ERR"
def test_vllmctl_has_no_removed_machinery():
source = open("/data/home/renbaibing/vllm/vllmctl", encoding="utf-8").read()
assert ".runas.py" not in source
assert ".user.env" not in source
assert "idle-watch on" not in source
assert "cmd_idle_watch" not in source
assert "admin/wake/" in source and "admin/sleep/" in source
assert os.access("/data/home/renbaibing/vllm/vllmctl", os.X_OK)

95
router/tests/test_singleflight.py Normal file
View File

@@ -0,0 +1,95 @@
"""Single-flight wake: N concurrent requests -> exactly one wake sequence."""
from __future__ import annotations
import asyncio
from config import DEPTH_AWAKE
async def test_ten_concurrent_requests_trigger_one_wake(pub, backend):
backend.set_sleeping("text", True, level=2)
backend.services["vllm-text"]["wake_delay"] = 0.15
async def one(i: int):
r = await pub.post("/v1/chat/completions", json={"model": "text", "seed": i})
assert r.status_code == 200, r.text
return r.json()
results = await asyncio.gather(*(one(i) for i in range(10)))
assert all(r["service"] == "vllm-text" for r in results)
svc = backend.services["vllm-text"]
assert svc["wake_up_calls"] == 1, backend.calls
assert svc["reload_calls"] == 1
assert svc["api_calls"] == 10
async def test_hold_timeout_does_not_cancel_the_wake(pub, backend, stack):
"""The first caller times out; the wake keeps going and finishes for the
next caller (no second wake sequence, no half-awake proxy)."""
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_delay"] = 0.3
stack.cfg.hold_offload_s = 0.05
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 503
assert r.json()["error"]["sleep_depth"] == "offloaded"
await asyncio.sleep(0.5) # let the shielded wake task finish
svc_state = stack.manager.services["text"]
assert svc_state.depth == DEPTH_AWAKE
assert backend.services["vllm-text"]["wake_up_calls"] == 1
r = await pub.post("/v1/chat/completions", json={"model": "text"})
assert r.status_code == 200
assert backend.services["vllm-text"]["wake_up_calls"] == 1 # cached awake
async def test_two_services_wake_concurrently_without_cross_talk(pub, backend):
backend.set_sleeping("text", True)
backend.set_sleeping("embed", True)
backend.services["vllm-text"]["wake_delay"] = 0.2
backend.services["vllm-embed"]["wake_delay"] = 0.05
chat = pub.post("/v1/chat/completions", json={"model": "text"})
embed = pub.post("/v1/embeddings", json={"input": "hi"})
r1, r2 = await asyncio.gather(chat, embed)
assert r1.status_code == 200 and r1.json()["service"] == "vllm-text"
assert r2.status_code == 200 and r2.json()["service"] == "vllm-embed"
assert backend.services["vllm-text"]["wake_up_calls"] == 1
assert backend.services["vllm-embed"]["wake_up_calls"] == 1
async def test_wake_state_is_cached_between_requests(pub, backend, stack):
stack.cfg.state_cache_ttl = 30.0
backend.set_sleeping("ocr", True)
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
probes_after_first = backend.count("GET vllm-ocr/is_sleeping")
assert probes_after_first >= 1
for _ in range(5):
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
# no extra probe per request while the awake state is cached
assert backend.count("GET vllm-ocr/is_sleeping") == probes_after_first
async def test_expired_cache_reprobes(pub, backend, stack):
stack.cfg.state_cache_ttl = 0.0
for _ in range(3):
assert (await pub.post("/v1/chat/completions", json={"model": "ocr"})).status_code == 200
assert backend.count("GET vllm-ocr/is_sleeping") >= 3
async def test_admin_and_request_path_share_one_lock(adm, pub, backend):
"""A wake driven from the admin port is joined by the public request path."""
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_delay"] = 0.2
admin_wake = asyncio.create_task(adm.post("/admin/wake/text"))
await asyncio.sleep(0.05)
r = await pub.post("/v1/chat/completions", json={"model": "text"})
admin_result = await admin_wake
assert r.status_code == 200
assert admin_result.status_code == 200
assert backend.services["vllm-text"]["wake_up_calls"] == 1

296
router/tests/test_wake_recovery.py Normal file
View File

@@ -0,0 +1,296 @@
"""Router-restart-mid-wake recovery (plan 6.4, E2E case 12).
`is_sleeping=false` does NOT mean "reload done": if the previous router died
between POST /wake_up and collective_rpc reload_weights, the backend happily
reports awake+healthy while serving garbage. The persisted wake-intent file
is what lets the new router tell the two apart.
"""
from __future__ import annotations
import asyncio
import json
import os
import httpx
from app import build_apps
from config import DEPTH_OFFLOADED, clone, load_config
from conftest import raw_asgi
from services import ServiceManager
def write_state(path, **services) -> None:
payload = {"version": 1, "updated": 0.0, "services": {
key: {"depth": value.get("depth"),
"wake_in_progress": value.get("wake_in_progress", False),
"level": value.get("level", 0),
"pending_reload": value.get("pending_reload", False)}
for key, value in services.items()
}}
with open(path, "w", encoding="utf-8") as fh:
json.dump(payload, fh)
def read_state(path) -> dict:
with open(path, encoding="utf-8") as fh:
return json.load(fh)["services"]
async def _make(cfg, backend, state_file):
public_app, admin_app, manager = build_apps(cfg, transport=backend)
try:
yield public_app, admin_app, manager
finally:
await manager.proxy_client.aclose()
await manager.ctrl_client.aclose()
async def test_startup_completes_interrupted_wake(tmp_path, backend):
"""The E2E failure: old router died after wake_up, before reload_weights.
The new router must not proxy until the sequence has been completed."""
state_file = str(tmp_path / "state.json")
write_state(state_file, text={"depth": "offloaded", "wake_in_progress": True,
"level": 2})
# backend *looks* perfectly healthy and awake -- that is the lie
backend.services["vllm-text"]["sleeping"] = False
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
cfg.state_cache_ttl = 0.0
async for _public, _admin, manager in _make(cfg, backend, state_file):
manager.start()
await asyncio.sleep(0.3) # let the recovery task run
text = manager.services["text"]
assert text.pending_reload is False
assert text.depth == "awake"
svc = backend.services["vllm-text"]
# the interrupted sequence was completed, not skipped
assert svc["reload_calls"] == 1
assert svc["reset_calls"] == 1
# and the file no longer claims a wake in flight
assert read_state(state_file)["text"]["wake_in_progress"] is False
async def test_interrupted_wake_never_proxies_before_recovery(tmp_path, backend, pub):
"""A request arriving in the recovery window must not be proxied on the
strength of is_sleeping=false alone."""
state_file = str(tmp_path / "state.json")
write_state(state_file, ocr={"depth": "offloaded", "wake_in_progress": True, "level": 2})
backend.services["vllm-ocr"]["sleeping"] = False # awake-looking
backend.services["vllm-ocr"]["api_delay"] = 0.0
backend.calls.clear()
stack_cfg = pub # unused; keeps the fixture ordering simple
del stack_cfg
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
async for public_app, _admin, manager in _make(cfg, backend, state_file):
manager.start()
# fire the request immediately, before recovery has finished probing
task = asyncio.create_task(_post(public_app, "/v1/chat/completions",
{"model": "ocr"}))
await asyncio.sleep(0.05)
ocr = backend.services["vllm-ocr"]
assert ocr["reload_calls"] >= 0
response = await task
assert response.status_code == 200
# proxied only after a completed reload sequence
assert backend.last("POST vllm-ocr/v1/chat/completions") > \
backend.last("POST vllm-ocr/reset_prefix_cache")
assert manager.services["ocr"].pending_reload is False
async def _post(app, path, payload) -> httpx.Response:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app),
base_url="http://router.test") as client:
return await client.post(path, json=payload)
async def test_startup_clears_state_when_unreachable(tmp_path, backend):
"""wake_in_progress + unreachable backend = fresh boot (weights are fresh);
nothing to complete."""
state_file = str(tmp_path / "state.json")
write_state(state_file, text={"depth": "offloaded", "wake_in_progress": True, "level": 2})
backend.services["vllm-text"]["reachable"] = False
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
async for _public, _admin, manager in _make(cfg, backend, state_file):
manager.start()
await asyncio.sleep(0.2)
text = manager.services["text"]
assert text.pending_reload is False
assert backend.services["vllm-text"]["reload_calls"] == 0
assert read_state(state_file)["text"]["wake_in_progress"] is False
async def test_startup_clears_state_when_still_sleeping(tmp_path, backend):
"""wake_in_progress but the backend never woke: the normal request path
will run a full wake when traffic arrives."""
state_file = str(tmp_path / "state.json")
write_state(state_file, embed={"depth": "sleeping", "wake_in_progress": True, "level": 1})
backend.set_sleeping("embed", True, level=1)
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
async for _public, _admin, manager in _make(cfg, backend, state_file):
manager.start()
await asyncio.sleep(0.2)
embed = manager.services["embed"]
assert embed.pending_reload is False
assert embed.depth == "sleeping" # restored from the file
assert backend.services["vllm-embed"]["reload_calls"] == 0
async def test_no_reload_when_no_wake_in_progress(tmp_path, backend, pub):
"""Plain restart with a settled backend: no reload, no state churn."""
state_file = str(tmp_path / "state.json")
write_state(state_file,
text={"depth": "awake", "wake_in_progress": False},
ocr={"depth": "offloaded", "wake_in_progress": False, "level": 2})
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
async for _public, _admin, manager in _make(cfg, backend, state_file):
manager.start()
await asyncio.sleep(0.1)
assert manager.services["ocr"].depth == "offloaded"
assert backend.count("POST vllm-ocr/wake_up") == 0
assert backend.count("POST vllm-text/collective_rpc") == 0
# the awake service serves immediately, straight through the proxy
status, body = await raw_asgi(_public_app(manager), "POST",
"/v1/chat/completions",
headers=[("content-type", "application/json")],
body=b'{"model": "text"}')
assert status == 200, body
svc = backend.services["vllm-text"]
assert svc["wake_up_calls"] == 0 # no pointless wake
assert svc["reload_calls"] == 0 # no pointless reload
def _public_app(manager):
from routing import build_public_app
return build_public_app(manager)
async def test_interrupted_wake_end_to_end(tmp_path, backend):
"""In-process replay of E2E case 12: the wake is killed after wake_up has
flipped is_sleeping but before reload_weights; a new manager on the same
state file must complete the sequence before proxying."""
state_file = str(tmp_path / "state.json")
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
cfg.wake_health_poll_s = 0.005
backend.set_sleeping("text", True, level=2)
backend.services["vllm-text"]["wake_delay"] = 0.4
backend.services["vllm-text"]["wake_flip_midway"] = True
first = ServiceManager(cfg, transport=backend)
try:
task = asyncio.create_task(first.ensure_awake("text"))
await asyncio.sleep(0.3) # is_sleeping flipped at t+0.2s
task.cancel() # ... and the router "dies"
await asyncio.gather(task, return_exceptions=True)
finally:
await first.proxy_client.aclose()
await first.ctrl_client.aclose()
assert backend.services["vllm-text"]["sleeping"] is False # looks awake
assert backend.services["vllm-text"]["reload_calls"] == 0 # ... but isn't
assert read_state(state_file)["text"]["wake_in_progress"] is True
backend.calls.clear()
async for public_app, _admin, manager in _make(cfg, backend, state_file):
manager.start()
response = await _post(public_app, "/v1/chat/completions", {"model": "text"})
assert response.status_code == 200
svc = backend.services["vllm-text"]
assert svc["reload_calls"] == 1 # sequence was completed ...
assert backend.last("POST vllm-text/v1/chat/completions") > \
backend.last("POST vllm-text/reset_prefix_cache") # ... first
assert manager.services["text"].pending_reload is False
async def test_wake_intent_is_written_and_cleared(tmp_path, backend):
"""The file says wake_in_progress=true while a wake is running and false
once it completes -- which is exactly what a restart in the middle leaves
behind."""
state_file = str(tmp_path / "state.json")
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
cfg.wake_health_poll_s = 0.005
backend.set_sleeping("text", True, level=2)
backend.services["vllm-text"]["wake_delay"] = 0.2
manager = ServiceManager(cfg, transport=backend)
try:
task = asyncio.create_task(manager.ensure_awake("text"))
await asyncio.sleep(0.05) # mid-wake
assert read_state(state_file)["text"]["wake_in_progress"] is True
outcome = await task
assert outcome.ok
entry = read_state(state_file)["text"]
assert entry["wake_in_progress"] is False
assert entry["depth"] == "awake"
finally:
await manager.proxy_client.aclose()
await manager.ctrl_client.aclose()
async def test_failed_wake_clears_the_intent(tmp_path, backend):
state_file = str(tmp_path / "state.json")
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
cfg.wake_health_poll_s = 0.005
backend.set_sleeping("text", True)
backend.services["vllm-text"]["wake_fails"] = 9
manager = ServiceManager(cfg, transport=backend)
try:
outcome = await manager.ensure_awake("text")
assert not outcome.ok
entry = read_state(state_file)["text"]
assert entry["wake_in_progress"] is False
finally:
await manager.proxy_client.aclose()
await manager.ctrl_client.aclose()
async def test_state_file_failure_is_not_fatal(tmp_path, backend):
"""An unwritable state path disables persistence, never serving."""
cfg = clone(load_config())
cfg.state_file = str(tmp_path / "no-such-dir" / "state.json")
cfg.idle_enabled = False
async for public_app, _admin, manager in _make(cfg, backend, cfg.state_file):
manager.start()
response = await _post(public_app, "/v1/chat/completions", {"model": "text"})
assert response.status_code == 200
assert not os.path.exists(cfg.state_file)
async def test_admin_reports_recovery(tmp_path, backend):
state_file = str(tmp_path / "state.json")
write_state(state_file, ocr={"depth": "offloaded", "wake_in_progress": True, "level": 2})
backend.services["vllm-ocr"]["sleeping"] = False
cfg = clone(load_config())
cfg.state_file = state_file
cfg.idle_enabled = False
async for _public, admin_app, manager in _make(cfg, backend, state_file):
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=admin_app),
base_url="http://router.test") as client:
r = await client.get("/admin/status")
assert r.status_code == 200
assert r.json()["services"]["ocr"]["wake_recovery_pending"] is True